feat: add more DinD

2025-12-29 00:25:53 +08:00
parent d536fbc995
commit 74cc6b49a7
12 changed files with 988 additions and 0 deletions
--- a/builds/kata-inside-dind/.env.example
+++ b/builds/kata-inside-dind/.env.example
@@ -0,0 +1,36 @@
+# Global Registry (optional)
+# GLOBAL_REGISTRY=registry.example.com/
+
+# Debian Version
+DEBIAN_VERSION=13.2
+
+# Kata Containers Version
+KATA_VERSION=3.24.0
+
+# Firecracker Version
+# Version of Firecracker VMM to install
+FIRECRACKER_VERSION=1.10.1
+
+# Kata DinD Image Version
+# Built image version tag
+KATA_DIND_VERSION=0.2.0
+
+# Timezone
+# Set the timezone for the container
+TZ=UTC
+
+# Kata Logging Level
+# Options: debug, info, warn, error
+KATA_LOGGING_LEVEL=info
+
+# Resource Limits
+# CPU limit (cores)
+KATA_DIND_CPU_LIMIT=2.00
+# Memory limit
+KATA_DIND_MEMORY_LIMIT=4G
+
+# Resource Reservations
+# CPU reservation (cores)
+KATA_DIND_CPU_RESERVATION=0.50
+# Memory reservation
+KATA_DIND_MEMORY_RESERVATION=1G
--- a/builds/kata-inside-dind/Dockerfile
+++ b/builds/kata-inside-dind/Dockerfile
@@ -0,0 +1,71 @@
+ARG DEBIAN_VERSION=13.2
+FROM debian:${DEBIAN_VERSION}
+
+# Install dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    gnupg \
+    iptables \
+    procps \
+    e2fsprogs \
+    xfsprogs \
+    xz-utils \
+    pigz \
+    zstd \
+    kmod \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Docker
+RUN install -m 0755 -d /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && chmod a+r /etc/apt/keyrings/docker.gpg \
+    && echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+    $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+    tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+    docker-ce \
+    docker-ce-cli \
+    containerd.io \
+    docker-buildx-plugin \
+    docker-compose-plugin \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Kata Containers (Static Binaries)
+ARG KATA_VERSION=3.24.0
+ARG ARCH=amd64
+RUN curl -fsSL https://github.com/kata-containers/kata-containers/releases/download/${KATA_VERSION}/kata-static-${KATA_VERSION}-${ARCH}.tar.zst -o kata-static.tar.zst \
+    && tar -I zstd -xvf kata-static.tar.zst -C / \
+    && rm kata-static.tar.zst \
+    && ln -s /opt/kata/bin/kata-runtime /usr/bin/kata-runtime \
+    && ln -s /opt/kata/bin/containerd-shim-kata-v2 /usr/bin/containerd-shim-kata-v2 \
+    && ln -s /opt/kata/bin/kata-monitor /usr/bin/kata-monitor \
+    && ln -s /opt/kata/bin/kata-collect-data /usr/bin/kata-collect-data \
+    && ln -s /opt/kata/bin/qemu-system-x86_64 /usr/bin/qemu-system-x86_64 || true \
+    && ln -s /opt/kata/libexec/virtiofsd /usr/bin/virtiofsd || true
+
+# Configure Kata
+RUN mkdir -p /etc/kata-containers \
+    && cp /opt/kata/share/defaults/kata-containers/configuration-qemu.toml /etc/kata-containers/ \
+    && cp /opt/kata/share/defaults/kata-containers/configuration-fc.toml /etc/kata-containers/ || true
+
+# Install Firecracker
+ARG FIRECRACKER_VERSION=1.14.0
+RUN if [ "${ARCH}" = "amd64" ]; then ARCH="x86_64"; fi \
+    && curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/v${FIRECRACKER_VERSION}/firecracker-v${FIRECRACKER_VERSION}-${ARCH}.tgz -o firecracker.tgz \
+    && tar -xzf firecracker.tgz \
+    && mv release-v${FIRECRACKER_VERSION}-${ARCH}/firecracker-v${FIRECRACKER_VERSION}-${ARCH} /usr/local/bin/firecracker \
+    && chmod +x /usr/local/bin/firecracker \
+    && rm -rf release-v${FIRECRACKER_VERSION}-${ARCH} firecracker.tgz
+
+# Set up dind
+VOLUME /var/lib/docker
+
+# Copy entrypoint script
+COPY dockerd-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh
+
+ENTRYPOINT ["dockerd-entrypoint.sh"]
+CMD ["dockerd"]
--- a/builds/kata-inside-dind/README.md
+++ b/builds/kata-inside-dind/README.md
@@ -0,0 +1,112 @@
+# Kata Containers inside Docker-in-Docker
+
+[中文文档](README.zh.md)
+
+A VM-based container runtime (Kata Containers with Firecracker VMM) running inside a Docker-in-Docker (DinD) container. This setup provides lightweight virtual machines with strong security isolation while maintaining container-like performance and simplicity. Kata Containers can use multiple hypervisors (QEMU, Firecracker, Cloud Hypervisor), and this image includes Firecracker by default for optimal performance.
+
+## Features
+
+- ✅ Complete Kata Containers runtime with official installation
+- ✅ Firecracker VMM for lightweight VM isolation
+- ✅ QEMU fallback support
+- ✅ Docker-in-Docker support for managing containers
+- ✅ VM-based container isolation with hardware security
+- ✅ Resource limits to prevent system exhaustion
+- ✅ Health checks for runtime readiness
+- ✅ Persistent storage for Kata and Docker data
+- ✅ Configurable logging levels
+- ✅ Compatible with Kubernetes via RuntimeClass
+
+## Prerequisites
+
+**Critical Requirements:**
+
+- Docker Engine 20.10+
+- Docker Compose 2.0+
+- **Host machine must support nested virtualization (KVM)**
+- `/dev/kvm` device available on the host
+- `/lib/modules` available on the host (for kernel module verification)
+- At least 2 CPU cores and 4GB RAM available
+- Privileged container support required
+
+### Verify Host Prerequisites
+
+```bash
+# Check if KVM is available
+ls -l /dev/kvm
+
+# For Intel CPUs, verify nested virtualization is enabled
+cat /sys/module/kvm_intel/parameters/nested
+# Should output 'Y' or '1'. If not:
+# sudo modprobe -r kvm_intel
+# sudo modprobe kvm_intel nested=1
+```
+
+## Quick Start
+
+1. Copy the environment file:
+
+   ```bash
+   cp .env.example .env
+   ```
+
+2. (Optional) Customize the configuration in `.env`
+
+3. Build and start the service:
+
+   ```bash
+   docker compose up -d --build
+   ```
+
+4. Wait for Kata runtime to be ready:
+
+   ```bash
+   docker compose logs -f kata-dind
+   ```
+
+5. Access the Docker daemon inside:
+
+   ```bash
+   # Get the container ID
+   docker compose ps
+
+   # Execute commands inside the container
+   docker compose exec kata-dind docker ps
+
+   # Run a container with Kata runtime (Firecracker)
+   docker compose exec kata-dind docker run --rm --runtime=kata-fc debian:bookworm uname -a
+
+   # Or run with QEMU (fallback)
+   docker compose exec kata-dind docker run --rm --runtime=kata debian:bookworm uname -a
+   ```
+
+## Configuration
+
+### Environment Variables
+
+| Variable                       | Default  | Description                                   |
+| ------------------------------ | -------- | --------------------------------------------- |
+| `DEBIAN_VERSION`               | `13.2`   | Base Debian version                           |
+| `KATA_VERSION`                 | `3.24.0` | Kata Containers version                       |
+| `FIRECRACKER_VERSION`          | `1.10.1` | Version of Firecracker VMM to install         |
+| `KATA_DIND_VERSION`            | `0.2.0`  | Built image version tag                       |
+| `TZ`                           | `UTC`    | Timezone for the container                    |
+| `KATA_LOGGING_LEVEL`           | `info`   | Kata logging level (debug, info, warn, error) |
+| `KATA_DIND_CPU_LIMIT`          | `2.00`   | CPU limit in cores                            |
+| `KATA_DIND_MEMORY_LIMIT`       | `4G`     | Memory limit                                  |
+| `KATA_DIND_CPU_RESERVATION`    | `0.50`   | CPU reservation in cores                      |
+| `KATA_DIND_MEMORY_RESERVATION` | `1G`     | Memory reservation                            |
+
+## Usage Examples
+
+### Running a Secure Container
+
+```bash
+docker compose exec kata-dind docker run -it --rm --runtime=kata-fc alpine sh
+```
+
+### Checking Runtime Information
+
+```bash
+docker compose exec kata-dind docker info | grep -i runtime
+```
--- a/builds/kata-inside-dind/README.zh.md
+++ b/builds/kata-inside-dind/README.zh.md
@@ -0,0 +1,112 @@
+# Kata 容器在 Docker-in-Docker 中运行
+
+[English Documentation](README.md)
+
+一个基于虚拟机的容器运行时（Kata Containers 与 Firecracker VMM）在 Docker-in-Docker（DinD）容器内运行。此设置提供轻量级虚拟机，具有强安全隔离的同时保持容器的性能和简洁性。Kata Containers 支持多种虚拟机管理器（QEMU、Firecracker、Cloud Hypervisor），此镜像默认包含 Firecracker 以获得最佳性能。
+
+## 特性
+
+- ✅ 使用官方安装脚本的完整 Kata Containers 运行时
+- ✅ Firecracker VMM 提供轻量级 VM 隔离
+- ✅ QEMU 回退支持
+- ✅ Docker-in-Docker 支持容器管理
+- ✅ 基于 VM 的容器隔离和硬件安全性
+- ✅ 资源限制防止系统资源耗尽
+- ✅ 运行时就绪性的健康检查
+- ✅ Kata 和 Docker 数据的持久化存储
+- ✅ 可配置的日志级别
+- ✅ 通过 RuntimeClass 与 Kubernetes 兼容
+
+## 前置条件
+
+**关键要求：**
+
+- Docker Engine 20.10+
+- Docker Compose 2.0+
+- **宿主机必须支持嵌套虚拟化（KVM）**
+- 宿主机上 `/dev/kvm` 设备可用
+- 宿主机上 `/lib/modules` 可用（用于内核模块验证）
+- 至少 2 个 CPU 核心和 4GB RAM
+- 需要特权容器支持
+
+### 验证宿主机前置条件
+
+```bash
+# 检查 KVM 是否可用
+ls -l /dev/kvm
+
+# 对于 Intel CPU，验证嵌套虚拟化是否启用
+cat /sys/module/kvm_intel/parameters/nested
+# 应该输出 'Y' 或 '1'。如果不是：
+# sudo modprobe -r kvm_intel
+# sudo modprobe kvm_intel nested=1
+```
+
+## 快速开始
+
+1. 复制环境文件：
+
+   ```bash
+   cp .env.example .env
+   ```
+
+2. （可选）在 `.env` 中自定义配置
+
+3. 构建并启动服务：
+
+   ```bash
+   docker compose up -d --build
+   ```
+
+4. 等待 Kata 运行时就绪：
+
+   ```bash
+   docker compose logs -f kata-dind
+   ```
+
+5. 访问容器内的 Docker 守护进程：
+
+   ```bash
+   # 获取容器 ID
+   docker compose ps
+
+   # 在容器内执行命令
+   docker compose exec kata-dind docker ps
+
+   # 使用 Firecracker 运行时运行容器
+   docker compose exec kata-dind docker run --rm --runtime=kata-fc debian:bookworm uname -a
+
+   # 或使用 QEMU（回退方案）
+   docker compose exec kata-dind docker run --rm --runtime=kata debian:bookworm uname -a
+   ```
+
+## 配置
+
+### 环境变量
+
+| 变量                           | 默认值   | 说明                                      |
+| ------------------------------ | -------- | ----------------------------------------- |
+| `DEBIAN_VERSION`               | `13.2`   | 基础 Debian 版本                          |
+| `KATA_VERSION`                 | `3.24.0` | Kata Containers 版本                      |
+| `FIRECRACKER_VERSION`          | `1.10.1` | 要安装的 Firecracker VMM 版本             |
+| `KATA_DIND_VERSION`            | `0.2.0`  | 构建的镜像版本标签                        |
+| `TZ`                           | `UTC`    | 容器的时区                                |
+| `KATA_LOGGING_LEVEL`           | `info`   | Kata 日志级别（debug、info、warn、error） |
+| `KATA_DIND_CPU_LIMIT`          | `2.00`   | CPU 限制（核心数）                        |
+| `KATA_DIND_MEMORY_LIMIT`       | `4G`     | 内存限制                                  |
+| `KATA_DIND_CPU_RESERVATION`    | `0.50`   | CPU 预留（核心数）                        |
+| `KATA_DIND_MEMORY_RESERVATION` | `1G`     | 内存预留                                  |
+
+## 使用示例
+
+### 运行安全容器
+
+```bash
+docker compose exec kata-dind docker run -it --rm --runtime=kata-fc alpine sh
+```
+
+### 检查运行时信息
+
+```bash
+docker compose exec kata-dind docker info | grep -i runtime
+```
--- a/builds/kata-inside-dind/docker-compose.yaml
+++ b/builds/kata-inside-dind/docker-compose.yaml
@@ -0,0 +1,55 @@
+# Kata Containers inside Docker-in-Docker
+# A VM-based container runtime running inside a Docker container with Docker daemon
+# See README.md for usage instructions
+
+x-defaults: &defaults
+  restart: unless-stopped
+  logging:
+    driver: json-file
+    options:
+      max-size: 100m
+      max-file: "3"
+
+services:
+  kata-dind:
+    <<: *defaults
+    image: ${GLOBAL_REGISTRY:-}alexsuntop/kata-inside-dind:${KATA_DIND_VERSION:-0.2.0}
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        DEBIAN_VERSION: ${DEBIAN_VERSION:-13.2}
+        KATA_VERSION: ${KATA_VERSION:-3.24.0}
+        FIRECRACKER_VERSION: ${FIRECRACKER_VERSION:-1.14.0}
+    privileged: true
+    devices:
+      - /dev/kvm:/dev/kvm
+      - /dev/net/tun:/dev/net/tun
+      - /dev/vhost-net:/dev/vhost-net
+      - /dev/vhost-vsock:/dev/vhost-vsock
+    volumes:
+      - kata_data:/var/lib/kata
+      - docker_data:/var/lib/docker
+      - /lib/modules:/lib/modules:ro
+    environment:
+      - TZ=${TZ:-UTC}
+      - DOCKER_TLS_CERTDIR=${DOCKER_TLS_CERTDIR:-}
+      - KATA_LOGGING_LEVEL=${KATA_LOGGING_LEVEL:-info}
+    healthcheck:
+      test: ["CMD", "docker", "info"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+    deploy:
+      resources:
+        limits:
+          cpus: ${KATA_DIND_CPU_LIMIT:-2.00}
+          memory: ${KATA_DIND_MEMORY_LIMIT:-4G}
+        reservations:
+          cpus: ${KATA_DIND_CPU_RESERVATION:-0.50}
+          memory: ${KATA_DIND_MEMORY_RESERVATION:-1G}
+
+volumes:
+  kata_data:
+  docker_data:
--- a/builds/kata-inside-dind/dockerd-entrypoint.sh
+++ b/builds/kata-inside-dind/dockerd-entrypoint.sh
@@ -0,0 +1,111 @@
+#!/bin/sh
+set -e
+
+# Adapted from official docker-library/docker dind entrypoint
+# https://github.com/docker-library/docker/blob/master/24/dind/dockerd-entrypoint.sh
+
+if [ -z "$DOCKER_HOST" ]; then
+  case "$1" in
+    dockerd*)
+      # If we're running dockerd, we need to make sure we have cgroups mounted
+      if [ ! -d /sys/fs/cgroup ]; then
+        mkdir -p /sys/fs/cgroup
+      fi
+      if ! mountpoint -q /sys/fs/cgroup; then
+        mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
+      fi
+
+      # Mount cgroup v2 if available and not mounted
+      if [ -e /sys/fs/cgroup/cgroup.controllers ] && ! mountpoint -q /sys/fs/cgroup; then
+        mount -t cgroup2 -o nsdelegate cgroup2 /sys/fs/cgroup
+      fi
+
+      # If /sys/fs/cgroup is not a cgroup2 mount, we might need to mount cgroup v1 hierarchies
+      if ! mountpoint -q /sys/fs/cgroup || [ "$(stat -f -c %T /sys/fs/cgroup)" != "cgroup2fs" ]; then
+        if [ -d /sys/fs/cgroup/cgroup.controllers ]; then
+          # It is cgroup2 but maybe not mounted as such?
+          # Actually if it exists, it's likely v2.
+          :
+        else
+          # cgroup v1
+          for subsystem in $(awk '/^[^#]/ { print $1 }' /proc/cgroups); do
+            mkdir -p "/sys/fs/cgroup/$subsystem"
+            if ! mountpoint -q "/sys/fs/cgroup/$subsystem"; then
+              mount -t cgroup -o "$subsystem" cgroup "/sys/fs/cgroup/$subsystem"
+            fi
+          done
+        fi
+      fi
+      ;;
+  esac
+fi
+
+# Configure Docker daemon with Kata runtime support
+mkdir -p /etc/docker
+cat > /etc/docker/daemon.json <<EOF
+{
+  "storage-driver": "overlay2",
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "10m",
+    "max-file": "3"
+  },
+  "runtimes": {
+    "kata": {
+      "path": "/usr/bin/containerd-shim-kata-v2",
+      "runtimeArgs": [
+        "--kata-config",
+        "/etc/kata-containers/configuration-qemu.toml"
+      ]
+    },
+    "kata-fc": {
+      "path": "/usr/bin/containerd-shim-kata-v2",
+      "runtimeArgs": [
+        "--kata-config",
+        "/etc/kata-containers/configuration-fc.toml"
+      ]
+    }
+  }
+}
+EOF
+
+# Configure Kata to use Firecracker if available
+if [ -f /etc/kata-containers/configuration-fc.toml ] && [ -x /usr/local/bin/firecracker ]; then
+    echo "[INFO] Firecracker configuration found"
+    # Update path in configuration if needed
+    if ! grep -q "path = \"/usr/local/bin/firecracker\"" /etc/kata-containers/configuration-fc.toml 2>/dev/null; then
+        if [ -w /etc/kata-containers/configuration-fc.toml ]; then
+            sed -i 's|path = ".*firecracker"|path = "/usr/local/bin/firecracker"|g' /etc/kata-containers/configuration-fc.toml || true
+        fi
+    fi
+fi
+
+# Enable debug logging if requested
+if [ "${KATA_LOGGING_LEVEL}" = "debug" ]; then
+    echo "[INFO] Enabling debug logging for Kata"
+    for config in /etc/kata-containers/configuration-*.toml; do
+        if [ -f "$config" ]; then
+            sed -i 's/enable_debug = false/enable_debug = true/g' "$config"
+            sed -i 's/#log_path = .*/log_path = "\/var\/log\/kata\/kata.log"/g' "$config"
+            sed -i 's/level = "info"/level = "debug"/g' "$config"
+        fi
+    done
+    mkdir -p /var/log/kata
+fi
+
+if [ "$1" = 'dockerd' ] || [ "${1#-}" != "$1" ]; then
+  # if the first argument is "dockerd" or a flag (starts with -)
+  if [ "${1#-}" != "$1" ]; then
+    set -- dockerd "$@"
+  fi
+
+    # Explicitly use iptables-legacy if available, as it is often more stable for DinD
+    if command -v update-alternatives >/dev/null; then
+        if update-alternatives --query iptables | grep -q "iptables-legacy"; then
+            update-alternatives --set iptables /usr/sbin/iptables-legacy || true
+            update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true
+        fi
+    fi
+fi
+
+exec "$@"