feat: add pulsar
This commit is contained in:
@@ -12,6 +12,7 @@ Compose Anything helps users quickly deploy various services by providing a set
|
|||||||
| [Apache HBase](./src/hbase) | 2.6 |
|
| [Apache HBase](./src/hbase) | 2.6 |
|
||||||
| [Apache HTTP Server](./src/apache) | 2.4.62 |
|
| [Apache HTTP Server](./src/apache) | 2.4.62 |
|
||||||
| [Apache Kafka](./src/kafka) | 7.8.0 |
|
| [Apache Kafka](./src/kafka) | 7.8.0 |
|
||||||
|
| [Apache Pulsar](./src/pulsar) | 4.0.7 |
|
||||||
| [Apache RocketMQ](./src/rocketmq) | 5.3.1 |
|
| [Apache RocketMQ](./src/rocketmq) | 5.3.1 |
|
||||||
| [Bifrost Gateway](./src/bifrost-gateway) | 1.2.15 |
|
| [Bifrost Gateway](./src/bifrost-gateway) | 1.2.15 |
|
||||||
| [Bolt.diy](./src/bolt-diy) | latest |
|
| [Bolt.diy](./src/bolt-diy) | latest |
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ Compose Anything 通过提供一组高质量的 Docker Compose 配置文件,
|
|||||||
| [Apache HBase](./src/hbase) | 2.6 |
|
| [Apache HBase](./src/hbase) | 2.6 |
|
||||||
| [Apache HTTP Server](./src/apache) | 2.4.62 |
|
| [Apache HTTP Server](./src/apache) | 2.4.62 |
|
||||||
| [Apache Kafka](./src/kafka) | 7.8.0 |
|
| [Apache Kafka](./src/kafka) | 7.8.0 |
|
||||||
|
| [Apache Pulsar](./src/pulsar) | 4.0.7 |
|
||||||
| [Apache RocketMQ](./src/rocketmq) | 5.3.1 |
|
| [Apache RocketMQ](./src/rocketmq) | 5.3.1 |
|
||||||
| [Bifrost Gateway](./src/bifrost-gateway) | 1.2.15 |
|
| [Bifrost Gateway](./src/bifrost-gateway) | 1.2.15 |
|
||||||
| [Bolt.diy](./src/bolt-diy) | latest |
|
| [Bolt.diy](./src/bolt-diy) | latest |
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ services:
|
|||||||
- TZ=${TZ:-UTC}
|
- TZ=${TZ:-UTC}
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
|
# healthcheck already built into the image
|
||||||
deploy:
|
deploy:
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
@@ -26,12 +27,6 @@ services:
|
|||||||
reservations:
|
reservations:
|
||||||
cpus: ${OPEN_WEBUI_CPU_RESERVATION:-0.1}
|
cpus: ${OPEN_WEBUI_CPU_RESERVATION:-0.1}
|
||||||
memory: ${OPEN_WEBUI_MEMORY_RESERVATION:-128M}
|
memory: ${OPEN_WEBUI_MEMORY_RESERVATION:-128M}
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/health"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 3
|
|
||||||
start_period: 30s
|
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
open_webui_data:
|
open_webui_data:
|
||||||
|
|||||||
84
src/pulsar/.env.example
Normal file
84
src/pulsar/.env.example
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
# Apache Pulsar version
|
||||||
|
PULSAR_VERSION=4.0.7
|
||||||
|
|
||||||
|
# Pulsar Manager version
|
||||||
|
PULSAR_MANAGER_VERSION=v0.4.0
|
||||||
|
|
||||||
|
# Timezone
|
||||||
|
TZ=UTC
|
||||||
|
|
||||||
|
# Global registry prefix (optional)
|
||||||
|
# GLOBAL_REGISTRY=your-registry.example.com/
|
||||||
|
|
||||||
|
# ==================== Port Overrides ====================
|
||||||
|
|
||||||
|
# Pulsar broker port (default: 6650)
|
||||||
|
# PULSAR_BROKER_PORT_OVERRIDE=6650
|
||||||
|
|
||||||
|
# Pulsar HTTP/Admin port (default: 8080)
|
||||||
|
# PULSAR_HTTP_PORT_OVERRIDE=8080
|
||||||
|
|
||||||
|
# Pulsar Manager web UI port (default: 9527)
|
||||||
|
# PULSAR_MANAGER_PORT_OVERRIDE=9527
|
||||||
|
|
||||||
|
# Pulsar Manager backend port (default: 7750)
|
||||||
|
# PULSAR_MANAGER_BACKEND_PORT_OVERRIDE=7750
|
||||||
|
|
||||||
|
# ==================== Standalone Mode Configuration ====================
|
||||||
|
|
||||||
|
# Enable ZooKeeper for standalone mode (0 = RocksDB, 1 = ZooKeeper)
|
||||||
|
# PULSAR_STANDALONE_USE_ZOOKEEPER=0
|
||||||
|
|
||||||
|
# JVM memory settings for standalone
|
||||||
|
# PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m
|
||||||
|
|
||||||
|
# ==================== Resource Limits (Standalone) ====================
|
||||||
|
|
||||||
|
# CPU limits
|
||||||
|
# PULSAR_CPU_LIMIT=2.00
|
||||||
|
# PULSAR_CPU_RESERVATION=0.50
|
||||||
|
|
||||||
|
# Memory limits
|
||||||
|
# PULSAR_MEMORY_LIMIT=2G
|
||||||
|
# PULSAR_MEMORY_RESERVATION=512M
|
||||||
|
|
||||||
|
# ==================== Cluster Mode Configuration ====================
|
||||||
|
|
||||||
|
# Cluster name
|
||||||
|
# PULSAR_CLUSTER_NAME=cluster-a
|
||||||
|
|
||||||
|
# ZooKeeper JVM memory settings
|
||||||
|
# ZOOKEEPER_MEM=-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m
|
||||||
|
|
||||||
|
# BookKeeper JVM memory settings
|
||||||
|
# BOOKIE_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m
|
||||||
|
|
||||||
|
# Broker JVM memory settings
|
||||||
|
# BROKER_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m
|
||||||
|
|
||||||
|
# ==================== Resource Limits (Cluster Mode) ====================
|
||||||
|
|
||||||
|
# ZooKeeper resources
|
||||||
|
# ZOOKEEPER_CPU_LIMIT=1.00
|
||||||
|
# ZOOKEEPER_CPU_RESERVATION=0.25
|
||||||
|
# ZOOKEEPER_MEMORY_LIMIT=512M
|
||||||
|
# ZOOKEEPER_MEMORY_RESERVATION=256M
|
||||||
|
|
||||||
|
# BookKeeper resources
|
||||||
|
# BOOKIE_CPU_LIMIT=1.00
|
||||||
|
# BOOKIE_CPU_RESERVATION=0.25
|
||||||
|
# BOOKIE_MEMORY_LIMIT=1G
|
||||||
|
# BOOKIE_MEMORY_RESERVATION=512M
|
||||||
|
|
||||||
|
# Broker resources
|
||||||
|
# BROKER_CPU_LIMIT=2.00
|
||||||
|
# BROKER_CPU_RESERVATION=0.50
|
||||||
|
# BROKER_MEMORY_LIMIT=2G
|
||||||
|
# BROKER_MEMORY_RESERVATION=512M
|
||||||
|
|
||||||
|
# ==================== Pulsar Manager Resources ====================
|
||||||
|
|
||||||
|
# PULSAR_MANAGER_CPU_LIMIT=1.00
|
||||||
|
# PULSAR_MANAGER_CPU_RESERVATION=0.25
|
||||||
|
# PULSAR_MANAGER_MEMORY_LIMIT=512M
|
||||||
|
# PULSAR_MANAGER_MEMORY_RESERVATION=256M
|
||||||
194
src/pulsar/README.md
Normal file
194
src/pulsar/README.md
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
# Apache Pulsar
|
||||||
|
|
||||||
|
[English](./README.md) | [中文](./README.zh.md)
|
||||||
|
|
||||||
|
Apache Pulsar is a cloud-native, distributed messaging and streaming platform. It combines the best features of traditional messaging systems like RabbitMQ with the high-throughput of stream processing systems like Kafka.
|
||||||
|
|
||||||
|
## Services
|
||||||
|
|
||||||
|
### Default (Standalone Mode)
|
||||||
|
|
||||||
|
- `pulsar`: Single-node Pulsar instance for development and testing.
|
||||||
|
|
||||||
|
### Cluster Mode (profile: `cluster`)
|
||||||
|
|
||||||
|
- `zookeeper`: ZooKeeper for cluster coordination.
|
||||||
|
- `pulsar-init`: Initializes cluster metadata (runs once).
|
||||||
|
- `bookie`: BookKeeper for persistent message storage.
|
||||||
|
- `broker`: Pulsar Broker for message routing.
|
||||||
|
|
||||||
|
### Management (profile: `manager`)
|
||||||
|
|
||||||
|
- `pulsar-manager`: Web UI for Pulsar cluster management.
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
| Variable Name | Description | Default Value |
|
||||||
|
| --------------------------------- | ---------------------------------------------- | ------------------------------------------------ |
|
||||||
|
| `PULSAR_VERSION` | Pulsar image version | `4.0.7` |
|
||||||
|
| `PULSAR_MANAGER_VERSION` | Pulsar Manager image version | `v0.4.0` |
|
||||||
|
| `TZ` | Timezone | `UTC` |
|
||||||
|
| `PULSAR_BROKER_PORT_OVERRIDE` | Host port for Pulsar broker (maps to 6650) | `6650` |
|
||||||
|
| `PULSAR_HTTP_PORT_OVERRIDE` | Host port for HTTP/Admin API (maps to 8080) | `8080` |
|
||||||
|
| `PULSAR_MANAGER_PORT_OVERRIDE` | Host port for Pulsar Manager UI (maps to 9527) | `9527` |
|
||||||
|
| `PULSAR_STANDALONE_USE_ZOOKEEPER` | Use ZooKeeper in standalone mode (0 or 1) | `0` |
|
||||||
|
| `PULSAR_MEM` | JVM memory settings for standalone | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` |
|
||||||
|
| `PULSAR_CLUSTER_NAME` | Cluster name (cluster mode) | `cluster-a` |
|
||||||
|
|
||||||
|
Please modify the `.env` file as needed for your use case.
|
||||||
|
|
||||||
|
## Volumes
|
||||||
|
|
||||||
|
- `pulsar_data`: Pulsar data directory (standalone mode).
|
||||||
|
- `pulsar_conf`: Pulsar configuration directory (standalone mode).
|
||||||
|
- `zookeeper_data`: ZooKeeper data directory (cluster mode).
|
||||||
|
- `bookie_data`: BookKeeper data directory (cluster mode).
|
||||||
|
- `pulsar_manager_data`: Pulsar Manager data directory.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Standalone Mode (Default)
|
||||||
|
|
||||||
|
1. Start Pulsar in standalone mode:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Access Pulsar:
|
||||||
|
- Broker: `pulsar://localhost:6650`
|
||||||
|
- Admin API: `http://localhost:8080`
|
||||||
|
|
||||||
|
### Cluster Mode
|
||||||
|
|
||||||
|
1. Start Pulsar cluster:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile cluster up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Wait for all services to be healthy:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile cluster ps
|
||||||
|
```
|
||||||
|
|
||||||
|
### With Pulsar Manager
|
||||||
|
|
||||||
|
1. Start with Pulsar Manager:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile manager up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Or with cluster mode:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile cluster --profile manager up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Initialize Pulsar Manager admin user (first time only):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CSRF_TOKEN=$(curl -s http://localhost:7750/pulsar-manager/csrf-token)
|
||||||
|
curl -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
|
||||||
|
-H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-X PUT http://localhost:7750/pulsar-manager/users/superuser \
|
||||||
|
-d '{"name": "admin", "password": "apachepulsar", "description": "admin user", "email": "admin@example.com"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Access Pulsar Manager at `http://localhost:9527`
|
||||||
|
- Default credentials: `admin` / `apachepulsar`
|
||||||
|
|
||||||
|
## Testing Pulsar
|
||||||
|
|
||||||
|
1. Create a namespace:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec pulsar bin/pulsar-admin namespaces create public/test-namespace
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Create a topic:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec pulsar bin/pulsar-admin topics create persistent://public/test-namespace/test-topic
|
||||||
|
```
|
||||||
|
|
||||||
|
3. List topics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec pulsar bin/pulsar-admin topics list public/test-namespace
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Produce messages:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it pulsar bin/pulsar-client produce persistent://public/test-namespace/test-topic --messages "Hello Pulsar"
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Consume messages:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it pulsar bin/pulsar-client consume persistent://public/test-namespace/test-topic -s "test-subscription" -n 0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Client Libraries
|
||||||
|
|
||||||
|
Pulsar supports multiple client libraries:
|
||||||
|
|
||||||
|
- Java: `org.apache.pulsar:pulsar-client`
|
||||||
|
- Python: `pip install pulsar-client`
|
||||||
|
- Go: `github.com/apache/pulsar-client-go`
|
||||||
|
- Node.js: `pulsar-client`
|
||||||
|
- C++: Native client available
|
||||||
|
|
||||||
|
Example (Python):
|
||||||
|
|
||||||
|
```python
|
||||||
|
import pulsar
|
||||||
|
|
||||||
|
client = pulsar.Client('pulsar://localhost:6650')
|
||||||
|
|
||||||
|
# Producer
|
||||||
|
producer = client.create_producer('persistent://public/default/my-topic')
|
||||||
|
producer.send('Hello Pulsar'.encode('utf-8'))
|
||||||
|
|
||||||
|
# Consumer
|
||||||
|
consumer = client.subscribe('persistent://public/default/my-topic', 'my-subscription')
|
||||||
|
msg = consumer.receive()
|
||||||
|
print(f"Received: {msg.data().decode('utf-8')}")
|
||||||
|
consumer.acknowledge(msg)
|
||||||
|
|
||||||
|
client.close()
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
- Standalone mode uses RocksDB as metadata store by default (recommended for single-node).
|
||||||
|
- Set `PULSAR_STANDALONE_USE_ZOOKEEPER=1` to use ZooKeeper as metadata store.
|
||||||
|
- Cluster mode is configured for single-node BookKeeper (ensemble size = 1).
|
||||||
|
- For production, adjust quorum settings and add more bookies.
|
||||||
|
|
||||||
|
## Ports
|
||||||
|
|
||||||
|
| Service | Port | Description |
|
||||||
|
| -------------- | ---- | --------------- |
|
||||||
|
| Pulsar Broker | 6650 | Binary protocol |
|
||||||
|
| Pulsar HTTP | 8080 | REST Admin API |
|
||||||
|
| Pulsar Manager | 9527 | Web UI |
|
||||||
|
| Pulsar Manager | 7750 | Backend API |
|
||||||
|
|
||||||
|
## Security Notes
|
||||||
|
|
||||||
|
- This configuration is for development/testing purposes.
|
||||||
|
- For production:
|
||||||
|
- Enable TLS encryption for broker connections.
|
||||||
|
- Configure authentication (JWT, OAuth2, etc.).
|
||||||
|
- Enable authorization with role-based access control.
|
||||||
|
- Use dedicated ZooKeeper and BookKeeper clusters.
|
||||||
|
- Regularly update Pulsar version for security patches.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
Apache Pulsar is licensed under the Apache License 2.0.
|
||||||
194
src/pulsar/README.zh.md
Normal file
194
src/pulsar/README.zh.md
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
# Apache Pulsar
|
||||||
|
|
||||||
|
[English](./README.md) | [中文](./README.zh.md)
|
||||||
|
|
||||||
|
Apache Pulsar 是一个云原生的分布式消息和流处理平台。它结合了传统消息系统(如 RabbitMQ)的最佳特性和流处理系统(如 Kafka)的高吞吐量优势。
|
||||||
|
|
||||||
|
## 服务
|
||||||
|
|
||||||
|
### 默认(单机模式)
|
||||||
|
|
||||||
|
- `pulsar`:单节点 Pulsar 实例,适用于开发和测试。
|
||||||
|
|
||||||
|
### 集群模式(profile: `cluster`)
|
||||||
|
|
||||||
|
- `zookeeper`:用于集群协调的 ZooKeeper。
|
||||||
|
- `pulsar-init`:初始化集群元数据(仅运行一次)。
|
||||||
|
- `bookie`:用于持久化消息存储的 BookKeeper。
|
||||||
|
- `broker`:用于消息路由的 Pulsar Broker。
|
||||||
|
|
||||||
|
### 管理工具(profile: `manager`)
|
||||||
|
|
||||||
|
- `pulsar-manager`:Pulsar 集群管理 Web UI。
|
||||||
|
|
||||||
|
## 环境变量
|
||||||
|
|
||||||
|
| 变量名 | 说明 | 默认值 |
|
||||||
|
| --------------------------------- | ----------------------------------------- | ------------------------------------------------ |
|
||||||
|
| `PULSAR_VERSION` | Pulsar 镜像版本 | `4.0.7` |
|
||||||
|
| `PULSAR_MANAGER_VERSION` | Pulsar Manager 镜像版本 | `v0.4.0` |
|
||||||
|
| `TZ` | 时区 | `UTC` |
|
||||||
|
| `PULSAR_BROKER_PORT_OVERRIDE` | Pulsar Broker 主机端口(映射到 6650) | `6650` |
|
||||||
|
| `PULSAR_HTTP_PORT_OVERRIDE` | HTTP/Admin API 主机端口(映射到 8080) | `8080` |
|
||||||
|
| `PULSAR_MANAGER_PORT_OVERRIDE` | Pulsar Manager UI 主机端口(映射到 9527) | `9527` |
|
||||||
|
| `PULSAR_STANDALONE_USE_ZOOKEEPER` | 单机模式使用 ZooKeeper(0 或 1) | `0` |
|
||||||
|
| `PULSAR_MEM` | 单机模式 JVM 内存设置 | `-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m` |
|
||||||
|
| `PULSAR_CLUSTER_NAME` | 集群名称(集群模式) | `cluster-a` |
|
||||||
|
|
||||||
|
请根据实际需求修改 `.env` 文件。
|
||||||
|
|
||||||
|
## 卷
|
||||||
|
|
||||||
|
- `pulsar_data`:Pulsar 数据目录(单机模式)。
|
||||||
|
- `pulsar_conf`:Pulsar 配置目录(单机模式)。
|
||||||
|
- `zookeeper_data`:ZooKeeper 数据目录(集群模式)。
|
||||||
|
- `bookie_data`:BookKeeper 数据目录(集群模式)。
|
||||||
|
- `pulsar_manager_data`:Pulsar Manager 数据目录。
|
||||||
|
|
||||||
|
## 使用方法
|
||||||
|
|
||||||
|
### 单机模式(默认)
|
||||||
|
|
||||||
|
1. 启动 Pulsar 单机模式:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
2. 访问 Pulsar:
|
||||||
|
- Broker:`pulsar://localhost:6650`
|
||||||
|
- Admin API:`http://localhost:8080`
|
||||||
|
|
||||||
|
### 集群模式
|
||||||
|
|
||||||
|
1. 启动 Pulsar 集群:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile cluster up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
2. 等待所有服务健康:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile cluster ps
|
||||||
|
```
|
||||||
|
|
||||||
|
### 使用 Pulsar Manager
|
||||||
|
|
||||||
|
1. 启动并包含 Pulsar Manager:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile manager up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
或者与集群模式一起使用:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --profile cluster --profile manager up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
2. 初始化 Pulsar Manager 管理员用户(仅首次):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CSRF_TOKEN=$(curl -s http://localhost:7750/pulsar-manager/csrf-token)
|
||||||
|
curl -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
|
||||||
|
-H "Cookie: XSRF-TOKEN=$CSRF_TOKEN" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-X PUT http://localhost:7750/pulsar-manager/users/superuser \
|
||||||
|
-d '{"name": "admin", "password": "apachepulsar", "description": "admin user", "email": "admin@example.com"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
3. 访问 Pulsar Manager:`http://localhost:9527`
|
||||||
|
- 默认凭据:`admin` / `apachepulsar`
|
||||||
|
|
||||||
|
## 测试 Pulsar
|
||||||
|
|
||||||
|
1. 创建命名空间:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec pulsar bin/pulsar-admin namespaces create public/test-namespace
|
||||||
|
```
|
||||||
|
|
||||||
|
2. 创建主题:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec pulsar bin/pulsar-admin topics create persistent://public/test-namespace/test-topic
|
||||||
|
```
|
||||||
|
|
||||||
|
3. 列出主题:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec pulsar bin/pulsar-admin topics list public/test-namespace
|
||||||
|
```
|
||||||
|
|
||||||
|
4. 生产消息:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it pulsar bin/pulsar-client produce persistent://public/test-namespace/test-topic --messages "Hello Pulsar"
|
||||||
|
```
|
||||||
|
|
||||||
|
5. 消费消息:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it pulsar bin/pulsar-client consume persistent://public/test-namespace/test-topic -s "test-subscription" -n 0
|
||||||
|
```
|
||||||
|
|
||||||
|
## 客户端库
|
||||||
|
|
||||||
|
Pulsar 支持多种客户端库:
|
||||||
|
|
||||||
|
- Java:`org.apache.pulsar:pulsar-client`
|
||||||
|
- Python:`pip install pulsar-client`
|
||||||
|
- Go:`github.com/apache/pulsar-client-go`
|
||||||
|
- Node.js:`pulsar-client`
|
||||||
|
- C++:原生客户端可用
|
||||||
|
|
||||||
|
示例(Python):
|
||||||
|
|
||||||
|
```python
|
||||||
|
import pulsar
|
||||||
|
|
||||||
|
client = pulsar.Client('pulsar://localhost:6650')
|
||||||
|
|
||||||
|
# 生产者
|
||||||
|
producer = client.create_producer('persistent://public/default/my-topic')
|
||||||
|
producer.send('Hello Pulsar'.encode('utf-8'))
|
||||||
|
|
||||||
|
# 消费者
|
||||||
|
consumer = client.subscribe('persistent://public/default/my-topic', 'my-subscription')
|
||||||
|
msg = consumer.receive()
|
||||||
|
print(f"收到消息: {msg.data().decode('utf-8')}")
|
||||||
|
consumer.acknowledge(msg)
|
||||||
|
|
||||||
|
client.close()
|
||||||
|
```
|
||||||
|
|
||||||
|
## 配置
|
||||||
|
|
||||||
|
- 单机模式默认使用 RocksDB 作为元数据存储(推荐用于单节点)。
|
||||||
|
- 设置 `PULSAR_STANDALONE_USE_ZOOKEEPER=1` 可使用 ZooKeeper 作为元数据存储。
|
||||||
|
- 集群模式配置为单节点 BookKeeper(ensemble size = 1)。
|
||||||
|
- 生产环境请调整 quorum 设置并添加更多 bookie。
|
||||||
|
|
||||||
|
## 端口
|
||||||
|
|
||||||
|
| 服务 | 端口 | 说明 |
|
||||||
|
| -------------- | ---- | -------------- |
|
||||||
|
| Pulsar Broker | 6650 | 二进制协议 |
|
||||||
|
| Pulsar HTTP | 8080 | REST Admin API |
|
||||||
|
| Pulsar Manager | 9527 | Web UI |
|
||||||
|
| Pulsar Manager | 7750 | 后端 API |
|
||||||
|
|
||||||
|
## 安全提示
|
||||||
|
|
||||||
|
- 此配置用于开发/测试目的。
|
||||||
|
- 生产环境请:
|
||||||
|
- 为 broker 连接启用 TLS 加密。
|
||||||
|
- 配置身份验证(JWT、OAuth2 等)。
|
||||||
|
- 启用基于角色的访问控制授权。
|
||||||
|
- 使用专用的 ZooKeeper 和 BookKeeper 集群。
|
||||||
|
- 定期更新 Pulsar 版本以获取安全补丁。
|
||||||
|
|
||||||
|
## 许可证
|
||||||
|
|
||||||
|
Apache Pulsar 采用 Apache License 2.0 许可证。
|
||||||
213
src/pulsar/docker-compose.yaml
Normal file
213
src/pulsar/docker-compose.yaml
Normal file
@@ -0,0 +1,213 @@
|
|||||||
|
# Apache Pulsar Docker Compose
|
||||||
|
# Provides standalone mode by default and cluster mode via profile
|
||||||
|
|
||||||
|
x-defaults: &defaults
|
||||||
|
restart: unless-stopped
|
||||||
|
logging:
|
||||||
|
driver: json-file
|
||||||
|
options:
|
||||||
|
max-size: 100m
|
||||||
|
max-file: "3"
|
||||||
|
|
||||||
|
services:
|
||||||
|
# Standalone mode (default) - single node deployment for development/testing
|
||||||
|
pulsar:
|
||||||
|
<<: *defaults
|
||||||
|
image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar:${PULSAR_VERSION:-4.0.7}
|
||||||
|
ports:
|
||||||
|
- "${PULSAR_BROKER_PORT_OVERRIDE:-6650}:6650"
|
||||||
|
- "${PULSAR_HTTP_PORT_OVERRIDE:-8080}:8080"
|
||||||
|
volumes:
|
||||||
|
- pulsar_data:/pulsar/data
|
||||||
|
- pulsar_conf:/pulsar/conf
|
||||||
|
environment:
|
||||||
|
- TZ=${TZ:-UTC}
|
||||||
|
- PULSAR_MEM=${PULSAR_MEM:--Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m}
|
||||||
|
- PULSAR_STANDALONE_USE_ZOOKEEPER=${PULSAR_STANDALONE_USE_ZOOKEEPER:-0}
|
||||||
|
command: bin/pulsar standalone
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: ${PULSAR_CPU_LIMIT:-2.00}
|
||||||
|
memory: ${PULSAR_MEMORY_LIMIT:-2G}
|
||||||
|
reservations:
|
||||||
|
cpus: ${PULSAR_CPU_RESERVATION:-0.50}
|
||||||
|
memory: ${PULSAR_MEMORY_RESERVATION:-512M}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "bin/pulsar-admin", "brokers", "healthcheck"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 5
|
||||||
|
start_period: 60s
|
||||||
|
|
||||||
|
# ==================== Cluster Mode (profile: cluster) ====================
|
||||||
|
|
||||||
|
# ZooKeeper for cluster coordination
|
||||||
|
zookeeper:
|
||||||
|
<<: *defaults
|
||||||
|
image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar:${PULSAR_VERSION:-4.0.7}
|
||||||
|
profiles:
|
||||||
|
- cluster
|
||||||
|
volumes:
|
||||||
|
- zookeeper_data:/pulsar/data/zookeeper
|
||||||
|
environment:
|
||||||
|
- TZ=${TZ:-UTC}
|
||||||
|
- metadataStoreUrl=zk:zookeeper:2181
|
||||||
|
- PULSAR_MEM=${ZOOKEEPER_MEM:--Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m}
|
||||||
|
command: >
|
||||||
|
bash -c "bin/apply-config-from-env.py conf/zookeeper.conf &&
|
||||||
|
bin/generate-zookeeper-config.sh conf/zookeeper.conf &&
|
||||||
|
exec bin/pulsar zookeeper"
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: ${ZOOKEEPER_CPU_LIMIT:-1.00}
|
||||||
|
memory: ${ZOOKEEPER_MEMORY_LIMIT:-512M}
|
||||||
|
reservations:
|
||||||
|
cpus: ${ZOOKEEPER_CPU_RESERVATION:-0.25}
|
||||||
|
memory: ${ZOOKEEPER_MEMORY_RESERVATION:-256M}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "bin/pulsar-zookeeper-ruok.sh"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 30
|
||||||
|
start_period: 10s
|
||||||
|
|
||||||
|
# Initialize cluster metadata
|
||||||
|
pulsar-init:
|
||||||
|
image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar:${PULSAR_VERSION:-4.0.7}
|
||||||
|
profiles:
|
||||||
|
- cluster
|
||||||
|
environment:
|
||||||
|
- TZ=${TZ:-UTC}
|
||||||
|
command: >
|
||||||
|
bash -c "bin/pulsar initialize-cluster-metadata
|
||||||
|
--cluster ${PULSAR_CLUSTER_NAME:-cluster-a}
|
||||||
|
--zookeeper zookeeper:2181
|
||||||
|
--configuration-store zookeeper:2181
|
||||||
|
--web-service-url http://broker:8080
|
||||||
|
--broker-service-url pulsar://broker:6650"
|
||||||
|
depends_on:
|
||||||
|
zookeeper:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
|
# BookKeeper (Bookie) for message storage
|
||||||
|
bookie:
|
||||||
|
<<: *defaults
|
||||||
|
image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar:${PULSAR_VERSION:-4.0.7}
|
||||||
|
profiles:
|
||||||
|
- cluster
|
||||||
|
volumes:
|
||||||
|
- bookie_data:/pulsar/data/bookkeeper
|
||||||
|
environment:
|
||||||
|
- TZ=${TZ:-UTC}
|
||||||
|
- clusterName=${PULSAR_CLUSTER_NAME:-cluster-a}
|
||||||
|
- zkServers=zookeeper:2181
|
||||||
|
- metadataServiceUri=metadata-store:zk:zookeeper:2181
|
||||||
|
- advertisedAddress=bookie
|
||||||
|
- BOOKIE_MEM=${BOOKIE_MEM:--Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m}
|
||||||
|
command: bash -c "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie"
|
||||||
|
depends_on:
|
||||||
|
zookeeper:
|
||||||
|
condition: service_healthy
|
||||||
|
pulsar-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: ${BOOKIE_CPU_LIMIT:-1.00}
|
||||||
|
memory: ${BOOKIE_MEMORY_LIMIT:-1G}
|
||||||
|
reservations:
|
||||||
|
cpus: ${BOOKIE_CPU_RESERVATION:-0.25}
|
||||||
|
memory: ${BOOKIE_MEMORY_RESERVATION:-512M}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "bin/bookkeeper", "shell", "bookiesanity"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 5
|
||||||
|
start_period: 30s
|
||||||
|
|
||||||
|
# Pulsar Broker for cluster mode
|
||||||
|
broker:
|
||||||
|
<<: *defaults
|
||||||
|
image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar:${PULSAR_VERSION:-4.0.7}
|
||||||
|
profiles:
|
||||||
|
- cluster
|
||||||
|
ports:
|
||||||
|
- "${PULSAR_BROKER_PORT_OVERRIDE:-6650}:6650"
|
||||||
|
- "${PULSAR_HTTP_PORT_OVERRIDE:-8080}:8080"
|
||||||
|
environment:
|
||||||
|
- TZ=${TZ:-UTC}
|
||||||
|
- metadataStoreUrl=zk:zookeeper:2181
|
||||||
|
- zookeeperServers=zookeeper:2181
|
||||||
|
- clusterName=${PULSAR_CLUSTER_NAME:-cluster-a}
|
||||||
|
- managedLedgerDefaultEnsembleSize=1
|
||||||
|
- managedLedgerDefaultWriteQuorum=1
|
||||||
|
- managedLedgerDefaultAckQuorum=1
|
||||||
|
- advertisedAddress=broker
|
||||||
|
- advertisedListeners=external:pulsar://127.0.0.1:6650
|
||||||
|
- PULSAR_MEM=${BROKER_MEM:--Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m}
|
||||||
|
command: bash -c "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker"
|
||||||
|
depends_on:
|
||||||
|
zookeeper:
|
||||||
|
condition: service_healthy
|
||||||
|
bookie:
|
||||||
|
condition: service_healthy
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: ${BROKER_CPU_LIMIT:-2.00}
|
||||||
|
memory: ${BROKER_MEMORY_LIMIT:-2G}
|
||||||
|
reservations:
|
||||||
|
cpus: ${BROKER_CPU_RESERVATION:-0.50}
|
||||||
|
memory: ${BROKER_MEMORY_RESERVATION:-512M}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "bin/pulsar-admin", "brokers", "healthcheck"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 5
|
||||||
|
start_period: 60s
|
||||||
|
|
||||||
|
# ==================== Pulsar Manager (profile: manager) ====================
|
||||||
|
|
||||||
|
pulsar-manager:
|
||||||
|
<<: *defaults
|
||||||
|
image: ${GLOBAL_REGISTRY:-}apachepulsar/pulsar-manager:${PULSAR_MANAGER_VERSION:-v0.4.0}
|
||||||
|
profiles:
|
||||||
|
- manager
|
||||||
|
ports:
|
||||||
|
- "${PULSAR_MANAGER_PORT_OVERRIDE:-9527}:9527"
|
||||||
|
- "${PULSAR_MANAGER_BACKEND_PORT_OVERRIDE:-7750}:7750"
|
||||||
|
volumes:
|
||||||
|
- pulsar_manager_data:/data
|
||||||
|
environment:
|
||||||
|
- TZ=${TZ:-UTC}
|
||||||
|
- SPRING_CONFIGURATION_FILE=/pulsar-manager/pulsar-manager/application.properties
|
||||||
|
- REDIRECT_HOST=http://127.0.0.1
|
||||||
|
- REDIRECT_PORT=${PULSAR_MANAGER_PORT_OVERRIDE:-9527}
|
||||||
|
- DRIVER_CLASS_NAME=org.postgresql.Driver
|
||||||
|
- URL=jdbc:postgresql://127.0.0.1:5432/pulsar_manager
|
||||||
|
- USERNAME=pulsar
|
||||||
|
- PASSWORD=pulsar
|
||||||
|
- LOG_LEVEL=DEBUG
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: ${PULSAR_MANAGER_CPU_LIMIT:-1.00}
|
||||||
|
memory: ${PULSAR_MANAGER_MEMORY_LIMIT:-512M}
|
||||||
|
reservations:
|
||||||
|
cpus: ${PULSAR_MANAGER_CPU_RESERVATION:-0.25}
|
||||||
|
memory: ${PULSAR_MANAGER_MEMORY_RESERVATION:-256M}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:7750/pulsar-manager/csrf-token"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 30s
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
pulsar_data:
|
||||||
|
pulsar_conf:
|
||||||
|
zookeeper_data:
|
||||||
|
bookie_data:
|
||||||
|
pulsar_manager_data:
|
||||||
Reference in New Issue
Block a user