# Restate Multi-Node Cluster
# A highly-available distributed workflow engine with 3-node cluster
# Documentation: https://docs.restate.dev/server/clusters

x-defaults: &defaults
  restart: unless-stopped
  logging:
    driver: json-file
    options:
      max-size: 100m
      max-file: '3'

x-environment: &common-env
  TZ: ${TZ:-UTC}
  RESTATE_CLUSTER_NAME: ${RESTATE_CLUSTER_NAME:-restate-cluster}
  RESTATE_LOG_FILTER: ${RESTATE_LOG_FILTER:-restate=info}
  # Minimum replicas required to accept writes (2 out of 3 nodes)
  RESTATE_DEFAULT_REPLICATION: ${RESTATE_DEFAULT_REPLICATION:-2}
  # Addresses where nodes can reach each other
  RESTATE_METADATA_CLIENT__ADDRESSES: ${RESTATE_METADATA_CLIENT__ADDRESSES:-["http://restate-1:5122","http://restate-2:5122","http://restate-3:5122"]}
  # Snapshot configuration
  RESTATE_WORKER__SNAPSHOTS__DESTINATION: ${RESTATE_WORKER__SNAPSHOTS__DESTINATION:-s3://restate/snapshots}
  RESTATE_WORKER__SNAPSHOTS__SNAPSHOT_INTERVAL_NUM_RECORDS: ${RESTATE_WORKER__SNAPSHOTS__SNAPSHOT_INTERVAL_NUM_RECORDS:-1000}
  RESTATE_WORKER__SNAPSHOTS__AWS_REGION: ${RESTATE_WORKER__SNAPSHOTS__AWS_REGION:-local}
  RESTATE_WORKER__SNAPSHOTS__AWS_ENDPOINT_URL: ${RESTATE_WORKER__SNAPSHOTS__AWS_ENDPOINT_URL:-http://minio:9000}
  RESTATE_WORKER__SNAPSHOTS__AWS_ALLOW_HTTP: ${RESTATE_WORKER__SNAPSHOTS__AWS_ALLOW_HTTP:-true}
  RESTATE_WORKER__SNAPSHOTS__AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-minioadmin}
  RESTATE_WORKER__SNAPSHOTS__AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-minioadmin}

x-restate: &restate-defaults
  <<: *defaults
  image: ${GLOBAL_REGISTRY:-}restatedev/restate:${RESTATE_VERSION:-1.5.3}
  volumes:
    - restate_data:/restate-data
  deploy:
    resources:
      limits:
        cpus: ${RESTATE_CLUSTER_CPU_LIMIT:-2.00}
        memory: ${RESTATE_CLUSTER_MEMORY_LIMIT:-2G}
      reservations:
        cpus: ${RESTATE_CLUSTER_CPU_RESERVATION:-0.50}
        memory: ${RESTATE_CLUSTER_MEMORY_RESERVATION:-512M}

services:
  restate-1:
    <<: *restate-defaults
    ports:
      - '${NODE1_INGRESS_PORT_OVERRIDE:-8080}:8080' # Ingress API
      - '${NODE1_ADMIN_PORT_OVERRIDE:-9070}:9070' # Admin API
      - '${NODE1_NODE_PORT_OVERRIDE:-5122}:5122' # Node-to-node
    environment:
      <<: *common-env
      RESTATE_NODE_NAME: restate-1
      RESTATE_FORCE_NODE_ID: 1
      RESTATE_ADVERTISED_ADDRESS: ${RESTATE_NODE1_ADVERTISED_ADDRESS:-http://restate-1:5122}
      RESTATE_AUTO_PROVISION: 'true' # Only first node provisions cluster
    healthcheck:
      test: [CMD, sh, -c, 'wget --no-verbose --tries=1 --spider http://localhost:9070/health || exit 1']
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    depends_on:
      minio:
        condition: service_healthy

  restate-2:
    <<: *restate-defaults
    ports:
      - '${NODE2_INGRESS_PORT_OVERRIDE:-28080}:8080'
      - '${NODE2_ADMIN_PORT_OVERRIDE:-29070}:9070'
      - '${NODE2_NODE_PORT_OVERRIDE:-25122}:5122'
    environment:
      <<: *common-env
      RESTATE_NODE_NAME: restate-2
      RESTATE_FORCE_NODE_ID: 2
      RESTATE_ADVERTISED_ADDRESS: ${RESTATE_NODE2_ADVERTISED_ADDRESS:-http://restate-2:5122}
      RESTATE_AUTO_PROVISION: 'false'
    healthcheck:
      test: [CMD, sh, -c, 'wget --no-verbose --tries=1 --spider http://localhost:9070/health || exit 1']
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    depends_on:
      minio:
        condition: service_healthy
      restate-1:
        condition: service_healthy

  restate-3:
    <<: *restate-defaults
    ports:
      - '${NODE3_INGRESS_PORT_OVERRIDE:-38080}:8080'
      - '${NODE3_ADMIN_PORT_OVERRIDE:-39070}:9070'
      - '${NODE3_NODE_PORT_OVERRIDE:-35122}:5122'
    environment:
      <<: *common-env
      RESTATE_NODE_NAME: restate-3
      RESTATE_FORCE_NODE_ID: 3
      RESTATE_ADVERTISED_ADDRESS: ${RESTATE_NODE3_ADVERTISED_ADDRESS:-http://restate-3:5122}
      RESTATE_AUTO_PROVISION: 'false'
    healthcheck:
      test: [CMD, sh, -c, 'wget --no-verbose --tries=1 --spider http://localhost:9070/health || exit 1']
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    depends_on:
      minio:
        condition: service_healthy
      restate-1:
        condition: service_healthy

  minio:
    <<: *defaults
    image: quay.io/minio/minio:${MINIO_VERSION:-latest}
    command: [sh, -c, 'mkdir -p /data/restate && /usr/bin/minio server --quiet /data --console-address :9001']
    ports:
      - '${MINIO_API_PORT_OVERRIDE:-9000}:9000'
      - '${MINIO_CONSOLE_PORT_OVERRIDE:-9001}:9001'
    environment:
      - MINIO_ROOT_USER=${MINIO_ROOT_USER:-minioadmin}
      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-minioadmin}
    volumes:
      - minio_data:/data
    healthcheck:
      test: [CMD, mc, ready, local]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    deploy:
      resources:
        limits:
          cpus: ${RESTATE_CLUSTER_MINIO_CPU_LIMIT:-1.00}
          memory: ${RESTATE_CLUSTER_MINIO_MEMORY_LIMIT:-512M}
        reservations:
          cpus: ${RESTATE_CLUSTER_MINIO_CPU_RESERVATION:-0.25}
          memory: ${RESTATE_CLUSTER_MINIO_MEMORY_RESERVATION:-128M}

volumes:
  restate_data:
  minio_data: