From 9a079fe79b6cd18dc33fe0fef5d55bf8dfe00910 Mon Sep 17 00:00:00 2001 From: Sun-ZhenXing <1006925066@qq.com> Date: Thu, 4 Dec 2025 11:59:24 +0800 Subject: [PATCH] feat: remove manager in pulsar --- src/pulsar/.env.example | 16 ------ src/pulsar/README.md | 102 +++++++++++++++++++-------------- src/pulsar/README.zh.md | 102 +++++++++++++++++++-------------- src/pulsar/docker-compose.yaml | 38 ------------ 4 files changed, 120 insertions(+), 138 deletions(-) diff --git a/src/pulsar/.env.example b/src/pulsar/.env.example index 92f70bc..55a8c7a 100644 --- a/src/pulsar/.env.example +++ b/src/pulsar/.env.example @@ -1,9 +1,6 @@ # Apache Pulsar version PULSAR_VERSION=4.0.7 -# Pulsar Manager version -PULSAR_MANAGER_VERSION=v0.4.0 - # Timezone TZ=UTC @@ -18,12 +15,6 @@ TZ=UTC # Pulsar HTTP/Admin port (default: 8080) # PULSAR_HTTP_PORT_OVERRIDE=8080 -# Pulsar Manager web UI port (default: 9527) -# PULSAR_MANAGER_PORT_OVERRIDE=9527 - -# Pulsar Manager backend port (default: 7750) -# PULSAR_MANAGER_BACKEND_PORT_OVERRIDE=7750 - # ==================== Standalone Mode Configuration ==================== # Enable ZooKeeper for standalone mode (0 = RocksDB, 1 = ZooKeeper) @@ -75,10 +66,3 @@ TZ=UTC # BROKER_CPU_RESERVATION=0.50 # BROKER_MEMORY_LIMIT=2G # BROKER_MEMORY_RESERVATION=512M - -# ==================== Pulsar Manager Resources ==================== - -# PULSAR_MANAGER_CPU_LIMIT=1.00 -# PULSAR_MANAGER_CPU_RESERVATION=0.25 -# PULSAR_MANAGER_MEMORY_LIMIT=512M -# PULSAR_MANAGER_MEMORY_RESERVATION=256M diff --git a/src/pulsar/README.md b/src/pulsar/README.md index cd2d12a..d9428cf 100644 --- a/src/pulsar/README.md +++ b/src/pulsar/README.md @@ -17,23 +17,17 @@ Apache Pulsar is a cloud-native, distributed messaging and streaming platform. I - `bookie`: BookKeeper for persistent message storage. - `broker`: Pulsar Broker for message routing. -### Management (profile: `manager`) - -- `pulsar-manager`: Web UI for Pulsar cluster management. - ## Environment Variables -| Variable Name | Description | Default Value | -| --------------------------------- | ---------------------------------------------- | ------------------------------------------------ | -| `PULSAR_VERSION` | Pulsar image version | `4.0.7` | -| `PULSAR_MANAGER_VERSION` | Pulsar Manager image version | `v0.4.0` | -| `TZ` | Timezone | `UTC` | -| `PULSAR_BROKER_PORT_OVERRIDE` | Host port for Pulsar broker (maps to 6650) | `6650` | -| `PULSAR_HTTP_PORT_OVERRIDE` | Host port for HTTP/Admin API (maps to 8080) | `8080` | -| `PULSAR_MANAGER_PORT_OVERRIDE` | Host port for Pulsar Manager UI (maps to 9527) | `9527` | -| `PULSAR_STANDALONE_USE_ZOOKEEPER` | Use ZooKeeper in standalone mode (0 or 1) | `0` | -| `PULSAR_MEM` | JVM memory settings for standalone | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` | -| `PULSAR_CLUSTER_NAME` | Cluster name (cluster mode) | `cluster-a` | +| Variable Name | Description | Default Value | +| --------------------------------- | ------------------------------------------- | ------------------------------------------------ | +| `PULSAR_VERSION` | Pulsar image version | `4.0.7` | +| `TZ` | Timezone | `UTC` | +| `PULSAR_BROKER_PORT_OVERRIDE` | Host port for Pulsar broker (maps to 6650) | `6650` | +| `PULSAR_HTTP_PORT_OVERRIDE` | Host port for HTTP/Admin API (maps to 8080) | `8080` | +| `PULSAR_STANDALONE_USE_ZOOKEEPER` | Use ZooKeeper in standalone mode (0 or 1) | `0` | +| `PULSAR_MEM` | JVM memory settings for standalone | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` | +| `PULSAR_CLUSTER_NAME` | Cluster name (cluster mode) | `cluster-a` | Please modify the `.env` file as needed for your use case. @@ -43,7 +37,6 @@ Please modify the `.env` file as needed for your use case. - `pulsar_conf`: Pulsar configuration directory (standalone mode). - `zookeeper_data`: ZooKeeper data directory (cluster mode). - `bookie_data`: BookKeeper data directory (cluster mode). -- `pulsar_manager_data`: Pulsar Manager data directory. ## Usage @@ -73,33 +66,60 @@ Please modify the `.env` file as needed for your use case. docker compose --profile cluster ps ``` -### With Pulsar Manager +## Management and Monitoring -1. Start with Pulsar Manager: +### Pulsar Admin CLI - ```bash - docker compose --profile manager up -d - ``` +The `pulsar-admin` CLI is the recommended tool for managing Pulsar. It's included in the Pulsar container. - Or with cluster mode: +```bash +# Check cluster health +docker exec pulsar bin/pulsar-admin brokers healthcheck - ```bash - docker compose --profile cluster --profile manager up -d - ``` +# List clusters +docker exec pulsar bin/pulsar-admin clusters list -2. Initialize Pulsar Manager admin user (first time only): +# List tenants +docker exec pulsar bin/pulsar-admin tenants list - ```bash - CSRF_TOKEN=$(curl -s http://localhost:7750/pulsar-manager/csrf-token) - curl -H "X-XSRF-TOKEN: $CSRF_TOKEN" \ - -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \ - -H "Content-Type: application/json" \ - -X PUT http://localhost:7750/pulsar-manager/users/superuser \ - -d '{"name": "admin", "password": "apachepulsar", "description": "admin user", "email": "admin@example.com"}' - ``` +# List namespaces +docker exec pulsar bin/pulsar-admin namespaces list public -3. Access Pulsar Manager at `http://localhost:9527` - - Default credentials: `admin` / `apachepulsar` +# Get broker stats +docker exec pulsar bin/pulsar-admin broker-stats monitoring-metrics +``` + +### REST Admin API + +Pulsar provides a comprehensive REST API for management tasks. + +```bash +# Get cluster information +curl http://localhost:8080/admin/v2/clusters + +# Get broker stats +curl http://localhost:8080/admin/v2/broker-stats/monitoring-metrics + +# List tenants +curl http://localhost:8080/admin/v2/tenants + +# List namespaces +curl http://localhost:8080/admin/v2/namespaces/public + +# Get topic stats +curl http://localhost:8080/admin/v2/persistent/public/default/my-topic/stats +``` + +### Monitoring with Prometheus + +Pulsar exposes Prometheus metrics at the `/metrics` endpoint: + +```bash +# Access Pulsar metrics +curl http://localhost:8080/metrics +``` + +You can integrate with Prometheus and Grafana for visualization. Pulsar provides official Grafana dashboards. ## Testing Pulsar @@ -172,12 +192,10 @@ client.close() ## Ports -| Service | Port | Description | -| -------------- | ---- | --------------- | -| Pulsar Broker | 6650 | Binary protocol | -| Pulsar HTTP | 8080 | REST Admin API | -| Pulsar Manager | 9527 | Web UI | -| Pulsar Manager | 7750 | Backend API | +| Service | Port | Description | +| ------------- | ---- | ------------------------ | +| Pulsar Broker | 6650 | Binary protocol | +| Pulsar HTTP | 8080 | REST Admin API & Metrics | ## Security Notes diff --git a/src/pulsar/README.zh.md b/src/pulsar/README.zh.md index fa091a2..2e43c3c 100644 --- a/src/pulsar/README.zh.md +++ b/src/pulsar/README.zh.md @@ -17,23 +17,17 @@ Apache Pulsar 是一个云原生的分布式消息和流处理平台。它结合 - `bookie`：用于持久化消息存储的 BookKeeper。 - `broker`：用于消息路由的 Pulsar Broker。 -### 管理工具（profile: `manager`） - -- `pulsar-manager`：Pulsar 集群管理 Web UI。 - ## 环境变量 -| 变量名 | 说明 | 默认值 | -| --------------------------------- | ----------------------------------------- | ------------------------------------------------ | -| `PULSAR_VERSION` | Pulsar 镜像版本 | `4.0.7` | -| `PULSAR_MANAGER_VERSION` | Pulsar Manager 镜像版本 | `v0.4.0` | -| `TZ` | 时区 | `UTC` | -| `PULSAR_BROKER_PORT_OVERRIDE` | Pulsar Broker 主机端口（映射到 6650） | `6650` | -| `PULSAR_HTTP_PORT_OVERRIDE` | HTTP/Admin API 主机端口（映射到 8080） | `8080` | -| `PULSAR_MANAGER_PORT_OVERRIDE` | Pulsar Manager UI 主机端口（映射到 9527） | `9527` | -| `PULSAR_STANDALONE_USE_ZOOKEEPER` | 单机模式使用 ZooKeeper（0 或 1） | `0` | -| `PULSAR_MEM` | 单机模式 JVM 内存设置 | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` | -| `PULSAR_CLUSTER_NAME` | 集群名称（集群模式） | `cluster-a` | +| 变量名 | 说明 | 默认值 | +| --------------------------------- | -------------------------------------- | ------------------------------------------------ | +| `PULSAR_VERSION` | Pulsar 镜像版本 | `4.0.7` | +| `TZ` | 时区 | `UTC` | +| `PULSAR_BROKER_PORT_OVERRIDE` | Pulsar Broker 主机端口（映射到 6650） | `6650` | +| `PULSAR_HTTP_PORT_OVERRIDE` | HTTP/Admin API 主机端口（映射到 8080） | `8080` | +| `PULSAR_STANDALONE_USE_ZOOKEEPER` | 单机模式使用 ZooKeeper（0 或 1） | `0` | +| `PULSAR_MEM` | 单机模式 JVM 内存设置 | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` | +| `PULSAR_CLUSTER_NAME` | 集群名称（集群模式） | `cluster-a` | 请根据实际需求修改 `.env` 文件。 @@ -43,7 +37,6 @@ Apache Pulsar 是一个云原生的分布式消息和流处理平台。它结合 - `pulsar_conf`：Pulsar 配置目录（单机模式）。 - `zookeeper_data`：ZooKeeper 数据目录（集群模式）。 - `bookie_data`：BookKeeper 数据目录（集群模式）。 -- `pulsar_manager_data`：Pulsar Manager 数据目录。 ## 使用方法 @@ -73,33 +66,60 @@ Apache Pulsar 是一个云原生的分布式消息和流处理平台。它结合 docker compose --profile cluster ps ``` -### 使用 Pulsar Manager +## 管理与监控 -1. 启动并包含 Pulsar Manager： +### Pulsar Admin CLI - ```bash - docker compose --profile manager up -d - ``` +`pulsar-admin` CLI 是管理 Pulsar 的推荐工具，已包含在 Pulsar 容器中。 - 或者与集群模式一起使用： +```bash +# 检查集群健康状态 +docker exec pulsar bin/pulsar-admin brokers healthcheck - ```bash - docker compose --profile cluster --profile manager up -d - ``` +# 列出集群 +docker exec pulsar bin/pulsar-admin clusters list -2. 初始化 Pulsar Manager 管理员用户（仅首次）： +# 列出租户 +docker exec pulsar bin/pulsar-admin tenants list - ```bash - CSRF_TOKEN=$(curl -s http://localhost:7750/pulsar-manager/csrf-token) - curl -H "X-XSRF-TOKEN: $CSRF_TOKEN" \ - -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \ - -H "Content-Type: application/json" \ - -X PUT http://localhost:7750/pulsar-manager/users/superuser \ - -d '{"name": "admin", "password": "apachepulsar", "description": "admin user", "email": "admin@example.com"}' - ``` +# 列出命名空间 +docker exec pulsar bin/pulsar-admin namespaces list public -3. 访问 Pulsar Manager：`http://localhost:9527` - - 默认凭据：`admin` / `apachepulsar` +# 获取 broker 统计信息 +docker exec pulsar bin/pulsar-admin broker-stats monitoring-metrics +``` + +### REST Admin API + +Pulsar 提供了全面的 REST API 用于管理任务。 + +```bash +# 获取集群信息 +curl http://localhost:8080/admin/v2/clusters + +# 获取 broker 统计信息 +curl http://localhost:8080/admin/v2/broker-stats/monitoring-metrics + +# 列出租户 +curl http://localhost:8080/admin/v2/tenants + +# 列出命名空间 +curl http://localhost:8080/admin/v2/namespaces/public + +# 获取主题统计信息 +curl http://localhost:8080/admin/v2/persistent/public/default/my-topic/stats +``` + +### 使用 Prometheus 监控 + +Pulsar 在 `/metrics` 端点暴露 Prometheus 指标： + +```bash +# 访问 Pulsar 指标 +curl http://localhost:8080/metrics +``` + +您可以集成 Prometheus 和 Grafana 进行可视化。Pulsar 提供了官方的 Grafana 仪表板。 ## 测试 Pulsar @@ -172,12 +192,10 @@ client.close() ## 端口 -| 服务 | 端口 | 说明 | -| -------------- | ---- | -------------- | -| Pulsar Broker | 6650 | 二进制协议 | -| Pulsar HTTP | 8080 | REST Admin API | -| Pulsar Manager | 9527 | Web UI | -| Pulsar Manager | 7750 | 后端 API | +| 服务 | 端口 | 说明 | +| ------------- | ---- | --------------------- | +| Pulsar Broker | 6650 | 二进制协议 | +| Pulsar HTTP | 8080 | REST Admin API 和指标 | ## 安全提示 diff --git a/src/pulsar/docker-compose.yaml b/src/pulsar/docker-compose.yaml index e5f1d2b..3ce7e6b 100644 --- a/src/pulsar/docker-compose.yaml +++ b/src/pulsar/docker-compose.yaml @@ -168,46 +168,8 @@ services: retries: 5 start_period: 60s - # ==================== Pulsar Manager (profile: manager) ==================== - - pulsar-manager: - <<: *defaults - image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar-manager:${PULSAR_MANAGER_VERSION:-v0.4.0} - profiles: - - manager - ports: - - "${PULSAR_MANAGER_PORT_OVERRIDE:-9527}:9527" - - "${PULSAR_MANAGER_BACKEND_PORT_OVERRIDE:-7750}:7750" - volumes: - - pulsar_manager_data:/data - environment: - - TZ=${TZ:-UTC} - - SPRING_CONFIGURATION_FILE=/pulsar-manager/pulsar-manager/application.properties - - REDIRECT_HOST=http://127.0.0.1 - - REDIRECT_PORT=${PULSAR_MANAGER_PORT_OVERRIDE:-9527} - - DRIVER_CLASS_NAME=org.postgresql.Driver - - URL=jdbc:postgresql://127.0.0.1:5432/pulsar_manager - - USERNAME=pulsar - - PASSWORD=pulsar - - LOG_LEVEL=DEBUG - deploy: - resources: - limits: - cpus: ${PULSAR_MANAGER_CPU_LIMIT:-1.00} - memory: ${PULSAR_MANAGER_MEMORY_LIMIT:-512M} - reservations: - cpus: ${PULSAR_MANAGER_CPU_RESERVATION:-0.25} - memory: ${PULSAR_MANAGER_MEMORY_RESERVATION:-256M} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:7750/pulsar-manager/csrf-token"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - volumes: pulsar_data: pulsar_conf: zookeeper_data: bookie_data: - pulsar_manager_data: