feat: add k3s-inside-dind

This commit is contained in:
Sun-ZhenXing
2025-12-25 17:54:03 +08:00
parent 1b34b74851
commit fe37a25c41
10 changed files with 559 additions and 13 deletions

View File

@@ -9,7 +9,7 @@ x-defaults: &defaults
services: services:
io-paint: io-paint:
<<: *defaults <<: *defaults
image: ${DOCKER_REGISTRY:-docker.io}/alexsuntop/io-paint:${BUILD_VERSION:-1.6.0} image: ${DOCKER_REGISTRY:-}alexsuntop/io-paint:${BUILD_VERSION:-1.6.0}
ports: ports:
- 8080:8080 - 8080:8080
build: build:

View File

@@ -0,0 +1,43 @@
# Global Registry (optional)
# GLOBAL_REGISTRY=registry.example.com/
# K3s Version
# Version of K3s to install
K3S_VERSION=v1.28.2+k3s1
# K3s DinD Image Version
# Built image version tag
K3S_DIND_VERSION=0.1.0
# Timezone
# Set the timezone for the container
TZ=UTC
# Kubernetes API Server Port
# Default: 6443
K3S_API_PORT_OVERRIDE=6443
# Docker TLS Port
# Default: 2376
DOCKER_TLS_PORT_OVERRIDE=2376
# K3s Token (optional)
# Shared secret token for cluster join
# K3S_TOKEN=
# K3s Disable Services
# Comma-separated list of services to disable
# Default: traefik
K3S_DISABLE_SERVICES=traefik
# Resource Limits
# CPU limit (cores)
K3S_DIND_CPU_LIMIT=2.00
# Memory limit
K3S_DIND_MEMORY_LIMIT=4G
# Resource Reservations
# CPU reservation (cores)
K3S_DIND_CPU_RESERVATION=0.50
# Memory reservation
K3S_DIND_MEMORY_RESERVATION=1G

1
builds/k3s-inside-dind/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
kubeconfig.yaml

View File

@@ -0,0 +1,21 @@
FROM docker:29-dind
ARG TARGETARCH=amd64
ARG K3S_VERSION=v1.28.2+k3s1
RUN apk add --no-cache bash iptables curl fuse-overlayfs
RUN if [ "$TARGETARCH" = "amd64" ]; then \
export SUFFIX=""; \
else \
export SUFFIX="-$TARGETARCH"; \
fi && \
curl -L -o /usr/local/bin/k3s https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s${SUFFIX} && \
chmod +x /usr/local/bin/k3s
EXPOSE 6443
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh
ENTRYPOINT ["entrypoint.sh"]

View File

@@ -0,0 +1,205 @@
# K3s inside Docker-in-Docker
[中文文档](README.zh.md)
A lightweight Kubernetes distribution (K3s) running inside a Docker-in-Docker (DinD) container. This setup allows you to run a complete Kubernetes cluster within a single Docker container, perfect for development, testing, and CI/CD pipelines.
## Features
- ✅ Complete K3s cluster in a single container
- ✅ Docker-in-Docker support for containerized workloads
- ✅ Kubernetes API server exposed on port 6443
- ✅ Multi-architecture support (x86-64, ARM64)
- ✅ Resource limits to prevent system exhaustion
- ✅ Health checks for cluster readiness
- ✅ Persistent storage for K3s and Docker data
## Prerequisites
- Docker Engine 20.10+
- Docker Compose 2.0+
- At least 2 CPU cores and 4GB RAM available
- Privileged container support
## Quick Start
1. Copy the environment file:
```bash
cp .env.example .env
```
2. (Optional) Customize the configuration in `.env`
3. Build and start the service:
```bash
docker compose up -d --build
```
4. Wait for K3s to be ready (check health status):
```bash
docker compose ps
```
5. Access the Kubernetes cluster:
```bash
# Copy kubeconfig from container
docker compose exec k3s cat /etc/rancher/k3s/k3s.yaml > kubeconfig.yaml
# Use kubectl with the config
export KUBECONFIG=$(pwd)/kubeconfig.yaml
kubectl get nodes
```
## Configuration
### Environment Variables
| Variable | Default | Description |
| ----------------------------- | -------------- | ------------------------------------- |
| `K3S_VERSION` | `v1.28.2+k3s1` | K3s version to install |
| `K3S_DIND_VERSION` | `0.1.0` | Built image version tag |
| `TZ` | `UTC` | Container timezone |
| `K3S_API_PORT_OVERRIDE` | `6443` | Kubernetes API server port |
| `DOCKER_TLS_PORT_OVERRIDE` | `2376` | Docker daemon TLS port |
| `K3S_TOKEN` | (empty) | Shared secret for cluster join |
| `K3S_DISABLE_SERVICES` | `traefik` | Services to disable (comma-separated) |
| `K3S_DIND_CPU_LIMIT` | `2.00` | CPU limit (cores) |
| `K3S_DIND_MEMORY_LIMIT` | `4G` | Memory limit |
| `K3S_DIND_CPU_RESERVATION` | `0.50` | CPU reservation (cores) |
| `K3S_DIND_MEMORY_RESERVATION` | `1G` | Memory reservation |
### Volumes
- `k3s_data`: K3s cluster data and state
- `docker_data`: Docker daemon data
## Usage Examples
### Deploy a Sample Application
```bash
# Create a deployment
docker compose exec k3s k3s kubectl create deployment nginx --image=nginx
# Expose it as a service
docker compose exec k3s k3s kubectl expose deployment nginx --port=80 --type=NodePort
# Check the service
docker compose exec k3s k3s kubectl get svc nginx
```
### Run Docker Commands Inside K3s
```bash
# Access the container
docker compose exec k3s sh
# Inside the container, you can use both docker and kubectl
docker ps
kubectl get pods -A
```
### Build and Deploy Custom Images
```bash
# Access the container
docker compose exec k3s sh
# Build an image inside the container
docker build -t myapp:latest .
# Deploy to K3s (using the local image)
kubectl create deployment myapp --image=myapp:latest
kubectl set image deployment/myapp myapp=myapp:latest --local -o yaml | kubectl apply -f -
```
## Security Considerations
⚠️ **Important Security Notes:**
- This container runs in **privileged mode**, which grants extensive system access
- Suitable for development and testing environments only
- **DO NOT** use in production without proper security hardening
- The Docker daemon inside is accessible without authentication by default
- All containers share the host's kernel
### Recommended for Production
For production workloads, consider:
- Running K3s natively on hosts or VMs
- Using managed Kubernetes services (EKS, GKE, AKS)
- Implementing proper network isolation
- Enabling RBAC and Pod Security Standards
- Using encrypted communication channels
## Troubleshooting
### Container Fails to Start
Check if your system supports privileged containers:
```bash
docker run --rm --privileged alpine sh -c "echo 'Privileged mode works'"
```
### K3s Server Not Ready
Wait longer for the cluster to initialize (60-90 seconds typically):
```bash
docker compose logs -f k3s
```
### kubectl Connection Refused
Ensure the kubeconfig server address points to `localhost` or the correct IP:
```bash
kubectl cluster-info
```
## Advanced Configuration
### Customize K3s Server Arguments
Modify the `entrypoint.sh` or pass environment variables to customize K3s behavior.
### Enable Additional K3s Services
By default, Traefik is disabled. To enable it:
```bash
# In .env file
K3S_DISABLE_SERVICES=
```
### Change K3s Version
Update the `K3S_VERSION` in `.env` and rebuild:
```bash
docker compose up -d --build
```
## Cleanup
Remove the cluster and all data:
```bash
docker compose down -v
```
## License
This configuration is provided as-is under the same license as the Compose Anything project.
## References
- [K3s Documentation](https://docs.k3s.io/)
- [Docker-in-Docker](https://hub.docker.com/_/docker)
- [Kubernetes Documentation](https://kubernetes.io/docs/)

View File

@@ -0,0 +1,205 @@
# K3s inside Docker-in-Docker
[English Documentation](README.md)
在 Docker-in-DockerDinD容器中运行的轻量级 Kubernetes 发行版K3s。此配置允许你在单个 Docker 容器内运行完整的 Kubernetes 集群,非常适合开发、测试和 CI/CD 流水线。
## 功能特性
- ✅ 在单个容器中运行完整的 K3s 集群
- ✅ 支持 Docker-in-Docker可运行容器化工作负载
- ✅ 在 6443 端口暴露 Kubernetes API 服务器
- ✅ 支持多架构x86-64、ARM64
- ✅ 资源限制防止系统资源耗尽
- ✅ 健康检查确保集群就绪
- ✅ 持久化存储 K3s 和 Docker 数据
## 前置要求
- Docker Engine 20.10+
- Docker Compose 2.0+
- 至少 2 个 CPU 核心和 4GB 内存
- 支持特权容器
## 快速开始
1. 复制环境变量文件:
```bash
cp .env.example .env
```
2. (可选)在 `.env` 中自定义配置
3. 构建并启动服务:
```bash
docker compose up -d --build
```
4. 等待 K3s 就绪(检查健康状态):
```bash
docker compose ps
```
5. 访问 Kubernetes 集群:
```bash
# 从容器中复制 kubeconfig
docker compose exec k3s cat /etc/rancher/k3s/k3s.yaml > kubeconfig.yaml
# 使用 kubectl 连接集群
export KUBECONFIG=$(pwd)/kubeconfig.yaml
kubectl get nodes
```
## 配置说明
### 环境变量
| 变量 | 默认值 | 说明 |
| ----------------------------- | -------------- | ------------------------- |
| `K3S_VERSION` | `v1.28.2+k3s1` | 要安装的 K3s 版本 |
| `K3S_DIND_VERSION` | `0.1.0` | 构建的镜像版本标签 |
| `TZ` | `UTC` | 容器时区 |
| `K3S_API_PORT_OVERRIDE` | `6443` | Kubernetes API 服务器端口 |
| `DOCKER_TLS_PORT_OVERRIDE` | `2376` | Docker 守护进程 TLS 端口 |
| `K3S_TOKEN` | (空) | 集群加入的共享密钥 |
| `K3S_DISABLE_SERVICES` | `traefik` | 要禁用的服务(逗号分隔) |
| `K3S_DIND_CPU_LIMIT` | `2.00` | CPU 限制(核心数) |
| `K3S_DIND_MEMORY_LIMIT` | `4G` | 内存限制 |
| `K3S_DIND_CPU_RESERVATION` | `0.50` | CPU 预留(核心数) |
| `K3S_DIND_MEMORY_RESERVATION` | `1G` | 内存预留 |
### 数据卷
- `k3s_data`K3s 集群数据和状态
- `docker_data`Docker 守护进程数据
## 使用示例
### 部署示例应用
```bash
# 创建部署
docker compose exec k3s k3s kubectl create deployment nginx --image=nginx
# 暴露为服务
docker compose exec k3s k3s kubectl expose deployment nginx --port=80 --type=NodePort
# 查看服务
docker compose exec k3s k3s kubectl get svc nginx
```
### 在 K3s 中运行 Docker 命令
```bash
# 进入容器
docker compose exec k3s sh
# 在容器内可以同时使用 docker 和 kubectl
docker ps
kubectl get pods -A
```
### 构建和部署自定义镜像
```bash
# 进入容器
docker compose exec k3s sh
# 在容器内构建镜像
docker build -t myapp:latest .
# 部署到 K3s使用本地镜像
kubectl create deployment myapp --image=myapp:latest
kubectl set image deployment/myapp myapp=myapp:latest --local -o yaml | kubectl apply -f -
```
## 安全注意事项
⚠️ **重要安全提示:**
- 此容器以**特权模式**运行,拥有广泛的系统访问权限
- 仅适用于开发和测试环境
- **请勿**在未经适当安全加固的情况下用于生产环境
- 容器内的 Docker 守护进程默认无需身份验证即可访问
- 所有容器共享主机的内核
### 生产环境建议
对于生产工作负载,请考虑:
- 在主机或虚拟机上原生运行 K3s
- 使用托管的 Kubernetes 服务EKS、GKE、AKS
- 实施适当的网络隔离
- 启用 RBAC 和 Pod 安全标准
- 使用加密通信通道
## 故障排除
### 容器启动失败
检查系统是否支持特权容器:
```bash
docker run --rm --privileged alpine sh -c "echo 'Privileged mode works'"
```
### K3s 服务器未就绪
等待更长时间让集群初始化(通常需要 60-90 秒):
```bash
docker compose logs -f k3s
```
### kubectl 连接被拒绝
确保 kubeconfig 中的服务器地址指向 `localhost` 或正确的 IP
```bash
kubectl cluster-info
```
## 高级配置
### 自定义 K3s 服务器参数
修改 `entrypoint.sh` 或传递环境变量来自定义 K3s 行为。
### 启用额外的 K3s 服务
默认情况下 Traefik 已禁用。要启用它:
```bash
# 在 .env 文件中
K3S_DISABLE_SERVICES=
```
### 更改 K3s 版本
在 `.env` 中更新 `K3S_VERSION` 并重新构建:
```bash
docker compose up -d --build
```
## 清理
删除集群和所有数据:
```bash
docker compose down -v
```
## 许可证
此配置按原样提供,遵循 Compose Anything 项目的相同许可证。
## 参考资料
- [K3s 文档](https://docs.k3s.io/)
- [Docker-in-Docker](https://hub.docker.com/_/docker)
- [Kubernetes 文档](https://kubernetes.io/docs/)

View File

@@ -0,0 +1,50 @@
# K3s inside Docker-in-Docker
# A lightweight Kubernetes cluster running inside a Docker container
# See README.md for usage instructions
x-defaults: &defaults
restart: unless-stopped
logging:
driver: json-file
options:
max-size: 100m
max-file: "3"
services:
k3s:
<<: *defaults
image: ${GLOBAL_REGISTRY:-}alexsuntop/k3s-inside-dind:${K3S_DIND_VERSION:-0.1.0}
build:
context: .
dockerfile: Dockerfile
args:
K3S_VERSION: ${K3S_VERSION:-v1.28.2+k3s1}
privileged: true
volumes:
- k3s_data:/var/lib/rancher/k3s
- docker_data:/var/lib/docker
ports:
- "${K3S_API_PORT_OVERRIDE:-6443}:6443" # Kubernetes API server
- "${DOCKER_TLS_PORT_OVERRIDE:-2376}:2376" # Docker daemon TLS port
environment:
- TZ=${TZ:-UTC}
- K3S_TOKEN=${K3S_TOKEN:-}
- K3S_DISABLE_SERVICES=${K3S_DISABLE_SERVICES:-traefik}
healthcheck:
test: ["CMD", "k3s", "kubectl", "get", "--raw", "/healthz"]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
deploy:
resources:
limits:
cpus: ${K3S_DIND_CPU_LIMIT:-2.00}
memory: ${K3S_DIND_MEMORY_LIMIT:-4G}
reservations:
cpus: ${K3S_DIND_CPU_RESERVATION:-0.50}
memory: ${K3S_DIND_MEMORY_RESERVATION:-1G}
volumes:
k3s_data:
docker_data:

View File

@@ -0,0 +1,25 @@
#!/bin/bash
set -e
dockerd-entrypoint.sh &
DOCKER_PID=$!
echo "Waiting for Docker daemon..."
timeout=30
while ! docker info > /dev/null 2>&1; do
timeout=$(($timeout - 1))
if [ $timeout -eq 0 ]; then
echo "Timed out waiting for Docker daemon to start"
exit 1
fi
sleep 1
done
echo "Docker is ready."
echo "Starting K3s..."
exec k3s server \
--snapshotter=native \
--disable=traefik \
--write-kubeconfig-mode=644 \
--https-listen-port=6443 \
"$@"

View File

@@ -10,7 +10,10 @@ MINIO_VERSION=latest
REDIS_VERSION=7 REDIS_VERSION=7
# Ports # Ports
LANGFUSE_PORT=3000 LANGFUSE_PORT_OVERRIDE=3000
LANGFUSE_WORKER_PORT_OVERRIDE=3030
MINIO_PORT_OVERRIDE=9090
MINIO_CONSOLE_PORT_OVERRIDE=9091
# PostgreSQL # PostgreSQL
POSTGRES_USER=postgres POSTGRES_USER=postgres

View File

@@ -29,7 +29,7 @@ services:
clickhouse: clickhouse:
condition: service_healthy condition: service_healthy
ports: ports:
- 127.0.0.1:3030:3030 - ${LANGFUSE_WORKER_PORT_OVERRIDE:-3030}:3030
environment: &langfuse-worker-env environment: &langfuse-worker-env
TZ: ${TZ:-UTC} TZ: ${TZ:-UTC}
NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000} NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000}
@@ -92,7 +92,7 @@ services:
image: ${GLOBAL_REGISTRY:-}langfuse/langfuse:${LANGFUSE_VERSION:-3} image: ${GLOBAL_REGISTRY:-}langfuse/langfuse:${LANGFUSE_VERSION:-3}
depends_on: *langfuse-depends-on depends_on: *langfuse-depends-on
ports: ports:
- "${LANGFUSE_PORT:-3000}:3000" - "${LANGFUSE_PORT_OVERRIDE:-3000}:3000"
environment: environment:
<<: *langfuse-worker-env <<: *langfuse-worker-env
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret}
@@ -133,8 +133,8 @@ services:
- langfuse_clickhouse_data:/var/lib/clickhouse - langfuse_clickhouse_data:/var/lib/clickhouse
- langfuse_clickhouse_logs:/var/log/clickhouse-server - langfuse_clickhouse_logs:/var/log/clickhouse-server
ports: ports:
- 127.0.0.1:8123:8123 - ${CLICKHOUSE_PORT_OVERRIDE:-8123}:8123
- 127.0.0.1:9000:9000 - ${CLICKHOUSE_TCP_PORT_OVERRIDE:-9000}:9000
healthcheck: healthcheck:
test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1 test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
interval: 5s interval: 5s
@@ -160,9 +160,6 @@ services:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio} MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret} MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret}
TZ: ${TZ:-UTC} TZ: ${TZ:-UTC}
ports:
- "9090:9000"
- 127.0.0.1:9091:9001
volumes: volumes:
- langfuse_minio_data:/data - langfuse_minio_data:/data
healthcheck: healthcheck:
@@ -186,8 +183,6 @@ services:
command: > command: >
--requirepass ${REDIS_AUTH:-myredissecret} --requirepass ${REDIS_AUTH:-myredissecret}
--maxmemory-policy noeviction --maxmemory-policy noeviction
ports:
- 127.0.0.1:6379:6379
healthcheck: healthcheck:
test: ["CMD", "redis-cli", "ping"] test: ["CMD", "redis-cli", "ping"]
interval: 3s interval: 3s
@@ -211,8 +206,6 @@ services:
POSTGRES_DB: ${POSTGRES_DB:-postgres} POSTGRES_DB: ${POSTGRES_DB:-postgres}
TZ: UTC TZ: UTC
PGTZ: UTC PGTZ: UTC
ports:
- 127.0.0.1:5432:5432
volumes: volumes:
- langfuse_postgres_data:/var/lib/postgresql/data - langfuse_postgres_data:/var/lib/postgresql/data
healthcheck: healthcheck: