feat: remove manager in pulsar
This commit is contained in:
@@ -1,9 +1,6 @@
|
|||||||
# Apache Pulsar version
|
# Apache Pulsar version
|
||||||
PULSAR_VERSION=4.0.7
|
PULSAR_VERSION=4.0.7
|
||||||
|
|
||||||
# Pulsar Manager version
|
|
||||||
PULSAR_MANAGER_VERSION=v0.4.0
|
|
||||||
|
|
||||||
# Timezone
|
# Timezone
|
||||||
TZ=UTC
|
TZ=UTC
|
||||||
|
|
||||||
@@ -18,12 +15,6 @@ TZ=UTC
|
|||||||
# Pulsar HTTP/Admin port (default: 8080)
|
# Pulsar HTTP/Admin port (default: 8080)
|
||||||
# PULSAR_HTTP_PORT_OVERRIDE=8080
|
# PULSAR_HTTP_PORT_OVERRIDE=8080
|
||||||
|
|
||||||
# Pulsar Manager web UI port (default: 9527)
|
|
||||||
# PULSAR_MANAGER_PORT_OVERRIDE=9527
|
|
||||||
|
|
||||||
# Pulsar Manager backend port (default: 7750)
|
|
||||||
# PULSAR_MANAGER_BACKEND_PORT_OVERRIDE=7750
|
|
||||||
|
|
||||||
# ==================== Standalone Mode Configuration ====================
|
# ==================== Standalone Mode Configuration ====================
|
||||||
|
|
||||||
# Enable ZooKeeper for standalone mode (0 = RocksDB, 1 = ZooKeeper)
|
# Enable ZooKeeper for standalone mode (0 = RocksDB, 1 = ZooKeeper)
|
||||||
@@ -75,10 +66,3 @@ TZ=UTC
|
|||||||
# BROKER_CPU_RESERVATION=0.50
|
# BROKER_CPU_RESERVATION=0.50
|
||||||
# BROKER_MEMORY_LIMIT=2G
|
# BROKER_MEMORY_LIMIT=2G
|
||||||
# BROKER_MEMORY_RESERVATION=512M
|
# BROKER_MEMORY_RESERVATION=512M
|
||||||
|
|
||||||
# ==================== Pulsar Manager Resources ====================
|
|
||||||
|
|
||||||
# PULSAR_MANAGER_CPU_LIMIT=1.00
|
|
||||||
# PULSAR_MANAGER_CPU_RESERVATION=0.25
|
|
||||||
# PULSAR_MANAGER_MEMORY_LIMIT=512M
|
|
||||||
# PULSAR_MANAGER_MEMORY_RESERVATION=256M
|
|
||||||
|
|||||||
@@ -17,20 +17,14 @@ Apache Pulsar is a cloud-native, distributed messaging and streaming platform. I
|
|||||||
- `bookie`: BookKeeper for persistent message storage.
|
- `bookie`: BookKeeper for persistent message storage.
|
||||||
- `broker`: Pulsar Broker for message routing.
|
- `broker`: Pulsar Broker for message routing.
|
||||||
|
|
||||||
### Management (profile: `manager`)
|
|
||||||
|
|
||||||
- `pulsar-manager`: Web UI for Pulsar cluster management.
|
|
||||||
|
|
||||||
## Environment Variables
|
## Environment Variables
|
||||||
|
|
||||||
| Variable Name | Description | Default Value |
|
| Variable Name | Description | Default Value |
|
||||||
| --------------------------------- | ---------------------------------------------- | ------------------------------------------------ |
|
| --------------------------------- | ------------------------------------------- | ------------------------------------------------ |
|
||||||
| `PULSAR_VERSION` | Pulsar image version | `4.0.7` |
|
| `PULSAR_VERSION` | Pulsar image version | `4.0.7` |
|
||||||
| `PULSAR_MANAGER_VERSION` | Pulsar Manager image version | `v0.4.0` |
|
|
||||||
| `TZ` | Timezone | `UTC` |
|
| `TZ` | Timezone | `UTC` |
|
||||||
| `PULSAR_BROKER_PORT_OVERRIDE` | Host port for Pulsar broker (maps to 6650) | `6650` |
|
| `PULSAR_BROKER_PORT_OVERRIDE` | Host port for Pulsar broker (maps to 6650) | `6650` |
|
||||||
| `PULSAR_HTTP_PORT_OVERRIDE` | Host port for HTTP/Admin API (maps to 8080) | `8080` |
|
| `PULSAR_HTTP_PORT_OVERRIDE` | Host port for HTTP/Admin API (maps to 8080) | `8080` |
|
||||||
| `PULSAR_MANAGER_PORT_OVERRIDE` | Host port for Pulsar Manager UI (maps to 9527) | `9527` |
|
|
||||||
| `PULSAR_STANDALONE_USE_ZOOKEEPER` | Use ZooKeeper in standalone mode (0 or 1) | `0` |
|
| `PULSAR_STANDALONE_USE_ZOOKEEPER` | Use ZooKeeper in standalone mode (0 or 1) | `0` |
|
||||||
| `PULSAR_MEM` | JVM memory settings for standalone | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` |
|
| `PULSAR_MEM` | JVM memory settings for standalone | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` |
|
||||||
| `PULSAR_CLUSTER_NAME` | Cluster name (cluster mode) | `cluster-a` |
|
| `PULSAR_CLUSTER_NAME` | Cluster name (cluster mode) | `cluster-a` |
|
||||||
@@ -43,7 +37,6 @@ Please modify the `.env` file as needed for your use case.
|
|||||||
- `pulsar_conf`: Pulsar configuration directory (standalone mode).
|
- `pulsar_conf`: Pulsar configuration directory (standalone mode).
|
||||||
- `zookeeper_data`: ZooKeeper data directory (cluster mode).
|
- `zookeeper_data`: ZooKeeper data directory (cluster mode).
|
||||||
- `bookie_data`: BookKeeper data directory (cluster mode).
|
- `bookie_data`: BookKeeper data directory (cluster mode).
|
||||||
- `pulsar_manager_data`: Pulsar Manager data directory.
|
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -73,33 +66,60 @@ Please modify the `.env` file as needed for your use case.
|
|||||||
docker compose --profile cluster ps
|
docker compose --profile cluster ps
|
||||||
```
|
```
|
||||||
|
|
||||||
### With Pulsar Manager
|
## Management and Monitoring
|
||||||
|
|
||||||
1. Start with Pulsar Manager:
|
### Pulsar Admin CLI
|
||||||
|
|
||||||
```bash
|
The `pulsar-admin` CLI is the recommended tool for managing Pulsar. It's included in the Pulsar container.
|
||||||
docker compose --profile manager up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
Or with cluster mode:
|
```bash
|
||||||
|
# Check cluster health
|
||||||
|
docker exec pulsar bin/pulsar-admin brokers healthcheck
|
||||||
|
|
||||||
```bash
|
# List clusters
|
||||||
docker compose --profile cluster --profile manager up -d
|
docker exec pulsar bin/pulsar-admin clusters list
|
||||||
```
|
|
||||||
|
|
||||||
2. Initialize Pulsar Manager admin user (first time only):
|
# List tenants
|
||||||
|
docker exec pulsar bin/pulsar-admin tenants list
|
||||||
|
|
||||||
```bash
|
# List namespaces
|
||||||
CSRF_TOKEN=$(curl -s http://localhost:7750/pulsar-manager/csrf-token)
|
docker exec pulsar bin/pulsar-admin namespaces list public
|
||||||
curl -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
|
|
||||||
-H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-X PUT http://localhost:7750/pulsar-manager/users/superuser \
|
|
||||||
-d '{"name": "admin", "password": "apachepulsar", "description": "admin user", "email": "admin@example.com"}'
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Access Pulsar Manager at `http://localhost:9527`
|
# Get broker stats
|
||||||
- Default credentials: `admin` / `apachepulsar`
|
docker exec pulsar bin/pulsar-admin broker-stats monitoring-metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
### REST Admin API
|
||||||
|
|
||||||
|
Pulsar provides a comprehensive REST API for management tasks.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Get cluster information
|
||||||
|
curl http://localhost:8080/admin/v2/clusters
|
||||||
|
|
||||||
|
# Get broker stats
|
||||||
|
curl http://localhost:8080/admin/v2/broker-stats/monitoring-metrics
|
||||||
|
|
||||||
|
# List tenants
|
||||||
|
curl http://localhost:8080/admin/v2/tenants
|
||||||
|
|
||||||
|
# List namespaces
|
||||||
|
curl http://localhost:8080/admin/v2/namespaces/public
|
||||||
|
|
||||||
|
# Get topic stats
|
||||||
|
curl http://localhost:8080/admin/v2/persistent/public/default/my-topic/stats
|
||||||
|
```
|
||||||
|
|
||||||
|
### Monitoring with Prometheus
|
||||||
|
|
||||||
|
Pulsar exposes Prometheus metrics at the `/metrics` endpoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Access Pulsar metrics
|
||||||
|
curl http://localhost:8080/metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
You can integrate with Prometheus and Grafana for visualization. Pulsar provides official Grafana dashboards.
|
||||||
|
|
||||||
## Testing Pulsar
|
## Testing Pulsar
|
||||||
|
|
||||||
@@ -173,11 +193,9 @@ client.close()
|
|||||||
## Ports
|
## Ports
|
||||||
|
|
||||||
| Service | Port | Description |
|
| Service | Port | Description |
|
||||||
| -------------- | ---- | --------------- |
|
| ------------- | ---- | ------------------------ |
|
||||||
| Pulsar Broker | 6650 | Binary protocol |
|
| Pulsar Broker | 6650 | Binary protocol |
|
||||||
| Pulsar HTTP | 8080 | REST Admin API |
|
| Pulsar HTTP | 8080 | REST Admin API & Metrics |
|
||||||
| Pulsar Manager | 9527 | Web UI |
|
|
||||||
| Pulsar Manager | 7750 | Backend API |
|
|
||||||
|
|
||||||
## Security Notes
|
## Security Notes
|
||||||
|
|
||||||
|
|||||||
@@ -17,20 +17,14 @@ Apache Pulsar 是一个云原生的分布式消息和流处理平台。它结合
|
|||||||
- `bookie`:用于持久化消息存储的 BookKeeper。
|
- `bookie`:用于持久化消息存储的 BookKeeper。
|
||||||
- `broker`:用于消息路由的 Pulsar Broker。
|
- `broker`:用于消息路由的 Pulsar Broker。
|
||||||
|
|
||||||
### 管理工具(profile: `manager`)
|
|
||||||
|
|
||||||
- `pulsar-manager`:Pulsar 集群管理 Web UI。
|
|
||||||
|
|
||||||
## 环境变量
|
## 环境变量
|
||||||
|
|
||||||
| 变量名 | 说明 | 默认值 |
|
| 变量名 | 说明 | 默认值 |
|
||||||
| --------------------------------- | ----------------------------------------- | ------------------------------------------------ |
|
| --------------------------------- | -------------------------------------- | ------------------------------------------------ |
|
||||||
| `PULSAR_VERSION` | Pulsar 镜像版本 | `4.0.7` |
|
| `PULSAR_VERSION` | Pulsar 镜像版本 | `4.0.7` |
|
||||||
| `PULSAR_MANAGER_VERSION` | Pulsar Manager 镜像版本 | `v0.4.0` |
|
|
||||||
| `TZ` | 时区 | `UTC` |
|
| `TZ` | 时区 | `UTC` |
|
||||||
| `PULSAR_BROKER_PORT_OVERRIDE` | Pulsar Broker 主机端口(映射到 6650) | `6650` |
|
| `PULSAR_BROKER_PORT_OVERRIDE` | Pulsar Broker 主机端口(映射到 6650) | `6650` |
|
||||||
| `PULSAR_HTTP_PORT_OVERRIDE` | HTTP/Admin API 主机端口(映射到 8080) | `8080` |
|
| `PULSAR_HTTP_PORT_OVERRIDE` | HTTP/Admin API 主机端口(映射到 8080) | `8080` |
|
||||||
| `PULSAR_MANAGER_PORT_OVERRIDE` | Pulsar Manager UI 主机端口(映射到 9527) | `9527` |
|
|
||||||
| `PULSAR_STANDALONE_USE_ZOOKEEPER` | 单机模式使用 ZooKeeper(0 或 1) | `0` |
|
| `PULSAR_STANDALONE_USE_ZOOKEEPER` | 单机模式使用 ZooKeeper(0 或 1) | `0` |
|
||||||
| `PULSAR_MEM` | 单机模式 JVM 内存设置 | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` |
|
| `PULSAR_MEM` | 单机模式 JVM 内存设置 | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` |
|
||||||
| `PULSAR_CLUSTER_NAME` | 集群名称(集群模式) | `cluster-a` |
|
| `PULSAR_CLUSTER_NAME` | 集群名称(集群模式) | `cluster-a` |
|
||||||
@@ -43,7 +37,6 @@ Apache Pulsar 是一个云原生的分布式消息和流处理平台。它结合
|
|||||||
- `pulsar_conf`:Pulsar 配置目录(单机模式)。
|
- `pulsar_conf`:Pulsar 配置目录(单机模式)。
|
||||||
- `zookeeper_data`:ZooKeeper 数据目录(集群模式)。
|
- `zookeeper_data`:ZooKeeper 数据目录(集群模式)。
|
||||||
- `bookie_data`:BookKeeper 数据目录(集群模式)。
|
- `bookie_data`:BookKeeper 数据目录(集群模式)。
|
||||||
- `pulsar_manager_data`:Pulsar Manager 数据目录。
|
|
||||||
|
|
||||||
## 使用方法
|
## 使用方法
|
||||||
|
|
||||||
@@ -73,33 +66,60 @@ Apache Pulsar 是一个云原生的分布式消息和流处理平台。它结合
|
|||||||
docker compose --profile cluster ps
|
docker compose --profile cluster ps
|
||||||
```
|
```
|
||||||
|
|
||||||
### 使用 Pulsar Manager
|
## 管理与监控
|
||||||
|
|
||||||
1. 启动并包含 Pulsar Manager:
|
### Pulsar Admin CLI
|
||||||
|
|
||||||
```bash
|
`pulsar-admin` CLI 是管理 Pulsar 的推荐工具,已包含在 Pulsar 容器中。
|
||||||
docker compose --profile manager up -d
|
|
||||||
```
|
|
||||||
|
|
||||||
或者与集群模式一起使用:
|
```bash
|
||||||
|
# 检查集群健康状态
|
||||||
|
docker exec pulsar bin/pulsar-admin brokers healthcheck
|
||||||
|
|
||||||
```bash
|
# 列出集群
|
||||||
docker compose --profile cluster --profile manager up -d
|
docker exec pulsar bin/pulsar-admin clusters list
|
||||||
```
|
|
||||||
|
|
||||||
2. 初始化 Pulsar Manager 管理员用户(仅首次):
|
# 列出租户
|
||||||
|
docker exec pulsar bin/pulsar-admin tenants list
|
||||||
|
|
||||||
```bash
|
# 列出命名空间
|
||||||
CSRF_TOKEN=$(curl -s http://localhost:7750/pulsar-manager/csrf-token)
|
docker exec pulsar bin/pulsar-admin namespaces list public
|
||||||
curl -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
|
|
||||||
-H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-X PUT http://localhost:7750/pulsar-manager/users/superuser \
|
|
||||||
-d '{"name": "admin", "password": "apachepulsar", "description": "admin user", "email": "admin@example.com"}'
|
|
||||||
```
|
|
||||||
|
|
||||||
3. 访问 Pulsar Manager:`http://localhost:9527`
|
# 获取 broker 统计信息
|
||||||
- 默认凭据:`admin` / `apachepulsar`
|
docker exec pulsar bin/pulsar-admin broker-stats monitoring-metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
### REST Admin API
|
||||||
|
|
||||||
|
Pulsar 提供了全面的 REST API 用于管理任务。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 获取集群信息
|
||||||
|
curl http://localhost:8080/admin/v2/clusters
|
||||||
|
|
||||||
|
# 获取 broker 统计信息
|
||||||
|
curl http://localhost:8080/admin/v2/broker-stats/monitoring-metrics
|
||||||
|
|
||||||
|
# 列出租户
|
||||||
|
curl http://localhost:8080/admin/v2/tenants
|
||||||
|
|
||||||
|
# 列出命名空间
|
||||||
|
curl http://localhost:8080/admin/v2/namespaces/public
|
||||||
|
|
||||||
|
# 获取主题统计信息
|
||||||
|
curl http://localhost:8080/admin/v2/persistent/public/default/my-topic/stats
|
||||||
|
```
|
||||||
|
|
||||||
|
### 使用 Prometheus 监控
|
||||||
|
|
||||||
|
Pulsar 在 `/metrics` 端点暴露 Prometheus 指标:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 访问 Pulsar 指标
|
||||||
|
curl http://localhost:8080/metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
您可以集成 Prometheus 和 Grafana 进行可视化。Pulsar 提供了官方的 Grafana 仪表板。
|
||||||
|
|
||||||
## 测试 Pulsar
|
## 测试 Pulsar
|
||||||
|
|
||||||
@@ -173,11 +193,9 @@ client.close()
|
|||||||
## 端口
|
## 端口
|
||||||
|
|
||||||
| 服务 | 端口 | 说明 |
|
| 服务 | 端口 | 说明 |
|
||||||
| -------------- | ---- | -------------- |
|
| ------------- | ---- | --------------------- |
|
||||||
| Pulsar Broker | 6650 | 二进制协议 |
|
| Pulsar Broker | 6650 | 二进制协议 |
|
||||||
| Pulsar HTTP | 8080 | REST Admin API |
|
| Pulsar HTTP | 8080 | REST Admin API 和指标 |
|
||||||
| Pulsar Manager | 9527 | Web UI |
|
|
||||||
| Pulsar Manager | 7750 | 后端 API |
|
|
||||||
|
|
||||||
## 安全提示
|
## 安全提示
|
||||||
|
|
||||||
|
|||||||
@@ -168,46 +168,8 @@ services:
|
|||||||
retries: 5
|
retries: 5
|
||||||
start_period: 60s
|
start_period: 60s
|
||||||
|
|
||||||
# ==================== Pulsar Manager (profile: manager) ====================
|
|
||||||
|
|
||||||
pulsar-manager:
|
|
||||||
<<: *defaults
|
|
||||||
image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar-manager:${PULSAR_MANAGER_VERSION:-v0.4.0}
|
|
||||||
profiles:
|
|
||||||
- manager
|
|
||||||
ports:
|
|
||||||
- "${PULSAR_MANAGER_PORT_OVERRIDE:-9527}:9527"
|
|
||||||
- "${PULSAR_MANAGER_BACKEND_PORT_OVERRIDE:-7750}:7750"
|
|
||||||
volumes:
|
|
||||||
- pulsar_manager_data:/data
|
|
||||||
environment:
|
|
||||||
- TZ=${TZ:-UTC}
|
|
||||||
- SPRING_CONFIGURATION_FILE=/pulsar-manager/pulsar-manager/application.properties
|
|
||||||
- REDIRECT_HOST=http://127.0.0.1
|
|
||||||
- REDIRECT_PORT=${PULSAR_MANAGER_PORT_OVERRIDE:-9527}
|
|
||||||
- DRIVER_CLASS_NAME=org.postgresql.Driver
|
|
||||||
- URL=jdbc:postgresql://127.0.0.1:5432/pulsar_manager
|
|
||||||
- USERNAME=pulsar
|
|
||||||
- PASSWORD=pulsar
|
|
||||||
- LOG_LEVEL=DEBUG
|
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpus: ${PULSAR_MANAGER_CPU_LIMIT:-1.00}
|
|
||||||
memory: ${PULSAR_MANAGER_MEMORY_LIMIT:-512M}
|
|
||||||
reservations:
|
|
||||||
cpus: ${PULSAR_MANAGER_CPU_RESERVATION:-0.25}
|
|
||||||
memory: ${PULSAR_MANAGER_MEMORY_RESERVATION:-256M}
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:7750/pulsar-manager/csrf-token"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 3
|
|
||||||
start_period: 30s
|
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
pulsar_data:
|
pulsar_data:
|
||||||
pulsar_conf:
|
pulsar_conf:
|
||||||
zookeeper_data:
|
zookeeper_data:
|
||||||
bookie_data:
|
bookie_data:
|
||||||
pulsar_manager_data:
|
|
||||||
|
|||||||
Reference in New Issue
Block a user