feat: add multica/

This commit is contained in:
Sun-ZhenXing
2026-04-14 15:22:06 +08:00
parent 41c4e8fd4e
commit ea1ca927c8
10 changed files with 787 additions and 10 deletions
+326
View File
@@ -0,0 +1,326 @@
#!/bin/bash
set -e
echo "==================== ClickHouse Initialization ===================="
clickhouse-client --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE}"
echo "✅ Database $CLICKHOUSE_DATABASE created successfully"
echo ""
echo "Creating OTEL tables required by OpenTelemetry Collector..."
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_traces
(
\`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`TraceId\` String CODEC(ZSTD(1)),
\`SpanId\` String CODEC(ZSTD(1)),
\`ParentSpanId\` String CODEC(ZSTD(1)),
\`TraceState\` String CODEC(ZSTD(1)),
\`SpanName\` LowCardinality(String) CODEC(ZSTD(1)),
\`SpanKind\` LowCardinality(String) CODEC(ZSTD(1)),
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ScopeName\` String CODEC(ZSTD(1)),
\`ScopeVersion\` String CODEC(ZSTD(1)),
\`SpanAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`Duration\` UInt64 CODEC(ZSTD(1)),
\`StatusCode\` LowCardinality(String) CODEC(ZSTD(1)),
\`StatusMessage\` String CODEC(ZSTD(1)),
\`Events.Timestamp\` Array(DateTime64(9)) CODEC(ZSTD(1)),
\`Events.Name\` Array(LowCardinality(String)) CODEC(ZSTD(1)),
\`Events.Attributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
\`Links.TraceId\` Array(String) CODEC(ZSTD(1)),
\`Links.SpanId\` Array(String) CODEC(ZSTD(1)),
\`Links.TraceState\` Array(String) CODEC(ZSTD(1)),
\`Links.Attributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_duration Duration TYPE minmax GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
TTL toDateTime(Timestamp) + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_logs
(
\`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`TimestampTime\` DateTime DEFAULT toDateTime(Timestamp),
\`TraceId\` String CODEC(ZSTD(1)),
\`SpanId\` String CODEC(ZSTD(1)),
\`TraceFlags\` UInt8,
\`SeverityText\` LowCardinality(String) CODEC(ZSTD(1)),
\`SeverityNumber\` UInt8,
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
\`Body\` String CODEC(ZSTD(1)),
\`ResourceSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ScopeSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
\`ScopeName\` String CODEC(ZSTD(1)),
\`ScopeVersion\` LowCardinality(String) CODEC(ZSTD(1)),
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`LogAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
PARTITION BY toDate(TimestampTime)
PRIMARY KEY (ServiceName, TimestampTime)
ORDER BY (ServiceName, TimestampTime, Timestamp)
TTL TimestampTime + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_metrics_gauge
(
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
\`ScopeName\` String CODEC(ZSTD(1)),
\`ScopeVersion\` String CODEC(ZSTD(1)),
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
\`MetricName\` String CODEC(ZSTD(1)),
\`MetricDescription\` String CODEC(ZSTD(1)),
\`MetricUnit\` String CODEC(ZSTD(1)),
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`Value\` Float64 CODEC(ZSTD(1)),
\`Flags\` UInt32 CODEC(ZSTD(1)),
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
TTL toDateTime(TimeUnix) + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_metrics_sum
(
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
\`ScopeName\` String CODEC(ZSTD(1)),
\`ScopeVersion\` String CODEC(ZSTD(1)),
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
\`MetricName\` String CODEC(ZSTD(1)),
\`MetricDescription\` String CODEC(ZSTD(1)),
\`MetricUnit\` String CODEC(ZSTD(1)),
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`Value\` Float64 CODEC(ZSTD(1)),
\`Flags\` UInt32 CODEC(ZSTD(1)),
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
\`AggregationTemporality\` Int32 CODEC(ZSTD(1)),
\`IsMonotonic\` Bool CODEC(Delta(1), ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
TTL toDateTime(TimeUnix) + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_metrics_histogram
(
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
\`ScopeName\` String CODEC(ZSTD(1)),
\`ScopeVersion\` String CODEC(ZSTD(1)),
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
\`MetricName\` String CODEC(ZSTD(1)),
\`MetricDescription\` String CODEC(ZSTD(1)),
\`MetricUnit\` String CODEC(ZSTD(1)),
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`Count\` UInt64 CODEC(Delta(8), ZSTD(1)),
\`Sum\` Float64 CODEC(ZSTD(1)),
\`BucketCounts\` Array(UInt64) CODEC(ZSTD(1)),
\`ExplicitBounds\` Array(Float64) CODEC(ZSTD(1)),
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
\`Flags\` UInt32 CODEC(ZSTD(1)),
\`Min\` Float64 CODEC(ZSTD(1)),
\`Max\` Float64 CODEC(ZSTD(1)),
\`AggregationTemporality\` Int32 CODEC(ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
TTL toDateTime(TimeUnix) + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_metrics_summary
(
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
\`ScopeName\` String CODEC(ZSTD(1)),
\`ScopeVersion\` String CODEC(ZSTD(1)),
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
\`MetricName\` String CODEC(ZSTD(1)),
\`MetricDescription\` String CODEC(ZSTD(1)),
\`MetricUnit\` String CODEC(ZSTD(1)),
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`Count\` UInt64 CODEC(Delta(8), ZSTD(1)),
\`Sum\` Float64 CODEC(ZSTD(1)),
\`ValueAtQuantiles.Quantile\` Array(Float64) CODEC(ZSTD(1)),
\`ValueAtQuantiles.Value\` Array(Float64) CODEC(ZSTD(1)),
\`Flags\` UInt32 CODEC(ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
TTL toDateTime(TimeUnix) + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_metrics_exponential_histogram
(
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
\`ScopeName\` String CODEC(ZSTD(1)),
\`ScopeVersion\` String CODEC(ZSTD(1)),
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
\`MetricName\` String CODEC(ZSTD(1)),
\`MetricDescription\` String CODEC(ZSTD(1)),
\`MetricUnit\` String CODEC(ZSTD(1)),
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
\`Count\` UInt64 CODEC(Delta(8), ZSTD(1)),
\`Sum\` Float64 CODEC(ZSTD(1)),
\`Scale\` Int32 CODEC(ZSTD(1)),
\`ZeroCount\` UInt64 CODEC(ZSTD(1)),
\`PositiveOffset\` Int32 CODEC(ZSTD(1)),
\`PositiveBucketCounts\` Array(UInt64) CODEC(ZSTD(1)),
\`NegativeOffset\` Int32 CODEC(ZSTD(1)),
\`NegativeBucketCounts\` Array(UInt64) CODEC(ZSTD(1)),
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
\`Flags\` UInt32 CODEC(ZSTD(1)),
\`Min\` Float64 CODEC(ZSTD(1)),
\`Max\` Float64 CODEC(ZSTD(1)),
\`AggregationTemporality\` Int32 CODEC(ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
TTL toDateTime(TimeUnix) + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts
(
\`TraceId\` String CODEC(ZSTD(1)),
\`Start\` DateTime CODEC(Delta(4), ZSTD(1)),
\`End\` DateTime CODEC(Delta(4), ZSTD(1)),
INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = MergeTree
PARTITION BY toDate(Start)
ORDER BY (TraceId, Start)
TTL toDateTime(Start) + toIntervalHour(730)
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
"
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv TO otel_traces_trace_id_ts
(
\`TraceId\` String,
\`Start\` DateTime64(9),
\`End\` DateTime64(9)
)
AS SELECT
TraceId,
min(Timestamp) AS Start,
max(Timestamp) AS End
FROM otel_traces
WHERE TraceId != ''
GROUP BY TraceId
"
echo "✅ All 9 OTEL tables created successfully"
echo "===================================================================="