feat: add multica/
This commit is contained in:
Sun-ZhenXing
@@ -0,0 +1,74 @@
|
||||
<?xml version="1.0"?>
|
||||
<clickhouse>
|
||||
<logger>
|
||||
<!-- Set console log level to warning (only critical messages) -->
|
||||
<level>warning</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
|
||||
<!-- Configure trace_log table settings -->
|
||||
<trace_log>
|
||||
<!-- Only log critical trace events (level 6 and above - more restrictive) -->
|
||||
<level>6</level>
|
||||
<!-- Reduce the frequency of trace log flushing -->
|
||||
<flush_interval_milliseconds>120000</flush_interval_milliseconds>
|
||||
<!-- Set table TTL to reduce storage (7 days) -->
|
||||
<table_ttl>604800</table_ttl>
|
||||
</trace_log>
|
||||
|
||||
<!-- Configure text_log table settings (also large in your case) -->
|
||||
<text_log>
|
||||
<!-- Only log warning level and above -->
|
||||
<level>warning</level>
|
||||
<!-- Set TTL to 7 days -->
|
||||
<table_ttl>604800</table_ttl>
|
||||
<!-- Reduce flush frequency -->
|
||||
<flush_interval_milliseconds>120000</flush_interval_milliseconds>
|
||||
</text_log>
|
||||
|
||||
<!-- Reduce other system table logging -->
|
||||
<query_log>
|
||||
<!-- Only log slow queries (over 1 second) -->
|
||||
<log_queries_min_query_duration_ms>1000</log_queries_min_query_duration_ms>
|
||||
<!-- Reduce flush frequency -->
|
||||
<flush_interval_milliseconds>60000</flush_interval_milliseconds>
|
||||
<!-- Set TTL to 7 days -->
|
||||
<table_ttl>604800</table_ttl>
|
||||
</query_log>
|
||||
|
||||
<!-- Configure system log levels -->
|
||||
<system_log>
|
||||
<level>warning</level>
|
||||
</system_log>
|
||||
|
||||
<!-- Reduce metric log verbosity -->
|
||||
<metric_log>
|
||||
<collect_interval_milliseconds>60000</collect_interval_milliseconds>
|
||||
<flush_interval_milliseconds>120000</flush_interval_milliseconds>
|
||||
<!-- Set TTL to 7 days -->
|
||||
<table_ttl>604800</table_ttl>
|
||||
</metric_log>
|
||||
|
||||
<!-- Configure asynchronous metric log (reduce storage) -->
|
||||
<asynchronous_metric_log>
|
||||
<collect_interval_milliseconds>60000</collect_interval_milliseconds>
|
||||
<flush_interval_milliseconds>120000</flush_interval_milliseconds>
|
||||
<!-- Set TTL to 7 days -->
|
||||
<table_ttl>604800</table_ttl>
|
||||
</asynchronous_metric_log>
|
||||
|
||||
<!-- Configure part log (reduce verbosity) -->
|
||||
<part_log>
|
||||
<level>warning</level>
|
||||
<flush_interval_milliseconds>120000</flush_interval_milliseconds>
|
||||
<!-- Set TTL to 7 days -->
|
||||
<table_ttl>604800</table_ttl>
|
||||
</part_log>
|
||||
|
||||
<!-- Configure latency log (reduce storage) -->
|
||||
<latency_log>
|
||||
<flush_interval_milliseconds>120000</flush_interval_milliseconds>
|
||||
<!-- Set TTL to 7 days -->
|
||||
<table_ttl>604800</table_ttl>
|
||||
</latency_log>
|
||||
</clickhouse>
|
||||
@@ -0,0 +1,326 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "==================== ClickHouse Initialization ===================="
|
||||
|
||||
|
||||
clickhouse-client --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE}"
|
||||
|
||||
echo "✅ Database $CLICKHOUSE_DATABASE created successfully"
|
||||
echo ""
|
||||
echo "Creating OTEL tables required by OpenTelemetry Collector..."
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_traces
|
||||
(
|
||||
\`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`TraceId\` String CODEC(ZSTD(1)),
|
||||
\`SpanId\` String CODEC(ZSTD(1)),
|
||||
\`ParentSpanId\` String CODEC(ZSTD(1)),
|
||||
\`TraceState\` String CODEC(ZSTD(1)),
|
||||
\`SpanName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`SpanKind\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ScopeName\` String CODEC(ZSTD(1)),
|
||||
\`ScopeVersion\` String CODEC(ZSTD(1)),
|
||||
\`SpanAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`Duration\` UInt64 CODEC(ZSTD(1)),
|
||||
\`StatusCode\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`StatusMessage\` String CODEC(ZSTD(1)),
|
||||
\`Events.Timestamp\` Array(DateTime64(9)) CODEC(ZSTD(1)),
|
||||
\`Events.Name\` Array(LowCardinality(String)) CODEC(ZSTD(1)),
|
||||
\`Events.Attributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
\`Links.TraceId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Links.SpanId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Links.TraceState\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Links.Attributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_duration Duration TYPE minmax GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(Timestamp)
|
||||
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
|
||||
TTL toDateTime(Timestamp) + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_logs
|
||||
(
|
||||
\`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`TimestampTime\` DateTime DEFAULT toDateTime(Timestamp),
|
||||
\`TraceId\` String CODEC(ZSTD(1)),
|
||||
\`SpanId\` String CODEC(ZSTD(1)),
|
||||
\`TraceFlags\` UInt8,
|
||||
\`SeverityText\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`SeverityNumber\` UInt8,
|
||||
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`Body\` String CODEC(ZSTD(1)),
|
||||
\`ResourceSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ScopeSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`ScopeName\` String CODEC(ZSTD(1)),
|
||||
\`ScopeVersion\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`LogAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimestampTime)
|
||||
PRIMARY KEY (ServiceName, TimestampTime)
|
||||
ORDER BY (ServiceName, TimestampTime, Timestamp)
|
||||
TTL TimestampTime + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_metrics_gauge
|
||||
(
|
||||
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ScopeName\` String CODEC(ZSTD(1)),
|
||||
\`ScopeVersion\` String CODEC(ZSTD(1)),
|
||||
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
|
||||
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`MetricName\` String CODEC(ZSTD(1)),
|
||||
\`MetricDescription\` String CODEC(ZSTD(1)),
|
||||
\`MetricUnit\` String CODEC(ZSTD(1)),
|
||||
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`Value\` Float64 CODEC(ZSTD(1)),
|
||||
\`Flags\` UInt32 CODEC(ZSTD(1)),
|
||||
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
|
||||
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_metrics_sum
|
||||
(
|
||||
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ScopeName\` String CODEC(ZSTD(1)),
|
||||
\`ScopeVersion\` String CODEC(ZSTD(1)),
|
||||
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
|
||||
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`MetricName\` String CODEC(ZSTD(1)),
|
||||
\`MetricDescription\` String CODEC(ZSTD(1)),
|
||||
\`MetricUnit\` String CODEC(ZSTD(1)),
|
||||
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`Value\` Float64 CODEC(ZSTD(1)),
|
||||
\`Flags\` UInt32 CODEC(ZSTD(1)),
|
||||
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
|
||||
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`AggregationTemporality\` Int32 CODEC(ZSTD(1)),
|
||||
\`IsMonotonic\` Bool CODEC(Delta(1), ZSTD(1)),
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_metrics_histogram
|
||||
(
|
||||
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ScopeName\` String CODEC(ZSTD(1)),
|
||||
\`ScopeVersion\` String CODEC(ZSTD(1)),
|
||||
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
|
||||
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`MetricName\` String CODEC(ZSTD(1)),
|
||||
\`MetricDescription\` String CODEC(ZSTD(1)),
|
||||
\`MetricUnit\` String CODEC(ZSTD(1)),
|
||||
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`Count\` UInt64 CODEC(Delta(8), ZSTD(1)),
|
||||
\`Sum\` Float64 CODEC(ZSTD(1)),
|
||||
\`BucketCounts\` Array(UInt64) CODEC(ZSTD(1)),
|
||||
\`ExplicitBounds\` Array(Float64) CODEC(ZSTD(1)),
|
||||
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
|
||||
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Flags\` UInt32 CODEC(ZSTD(1)),
|
||||
\`Min\` Float64 CODEC(ZSTD(1)),
|
||||
\`Max\` Float64 CODEC(ZSTD(1)),
|
||||
\`AggregationTemporality\` Int32 CODEC(ZSTD(1)),
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_metrics_summary
|
||||
(
|
||||
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ScopeName\` String CODEC(ZSTD(1)),
|
||||
\`ScopeVersion\` String CODEC(ZSTD(1)),
|
||||
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
|
||||
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`MetricName\` String CODEC(ZSTD(1)),
|
||||
\`MetricDescription\` String CODEC(ZSTD(1)),
|
||||
\`MetricUnit\` String CODEC(ZSTD(1)),
|
||||
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`Count\` UInt64 CODEC(Delta(8), ZSTD(1)),
|
||||
\`Sum\` Float64 CODEC(ZSTD(1)),
|
||||
\`ValueAtQuantiles.Quantile\` Array(Float64) CODEC(ZSTD(1)),
|
||||
\`ValueAtQuantiles.Value\` Array(Float64) CODEC(ZSTD(1)),
|
||||
\`Flags\` UInt32 CODEC(ZSTD(1)),
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_metrics_exponential_histogram
|
||||
(
|
||||
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ResourceSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ScopeName\` String CODEC(ZSTD(1)),
|
||||
\`ScopeVersion\` String CODEC(ZSTD(1)),
|
||||
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`ScopeDroppedAttrCount\` UInt32 CODEC(ZSTD(1)),
|
||||
\`ScopeSchemaUrl\` String CODEC(ZSTD(1)),
|
||||
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
|
||||
\`MetricName\` String CODEC(ZSTD(1)),
|
||||
\`MetricDescription\` String CODEC(ZSTD(1)),
|
||||
\`MetricUnit\` String CODEC(ZSTD(1)),
|
||||
\`Attributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
|
||||
\`StartTimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`TimeUnix\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
|
||||
\`Count\` UInt64 CODEC(Delta(8), ZSTD(1)),
|
||||
\`Sum\` Float64 CODEC(ZSTD(1)),
|
||||
\`Scale\` Int32 CODEC(ZSTD(1)),
|
||||
\`ZeroCount\` UInt64 CODEC(ZSTD(1)),
|
||||
\`PositiveOffset\` Int32 CODEC(ZSTD(1)),
|
||||
\`PositiveBucketCounts\` Array(UInt64) CODEC(ZSTD(1)),
|
||||
\`NegativeOffset\` Int32 CODEC(ZSTD(1)),
|
||||
\`NegativeBucketCounts\` Array(UInt64) CODEC(ZSTD(1)),
|
||||
\`Exemplars.FilteredAttributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TimeUnix\` Array(DateTime64(9)) CODEC(ZSTD(1)),
|
||||
\`Exemplars.Value\` Array(Float64) CODEC(ZSTD(1)),
|
||||
\`Exemplars.SpanId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Exemplars.TraceId\` Array(String) CODEC(ZSTD(1)),
|
||||
\`Flags\` UInt32 CODEC(ZSTD(1)),
|
||||
\`Min\` Float64 CODEC(ZSTD(1)),
|
||||
\`Max\` Float64 CODEC(ZSTD(1)),
|
||||
\`AggregationTemporality\` Int32 CODEC(ZSTD(1)),
|
||||
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
|
||||
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(TimeUnix)
|
||||
ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))
|
||||
TTL toDateTime(TimeUnix) + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts
|
||||
(
|
||||
\`TraceId\` String CODEC(ZSTD(1)),
|
||||
\`Start\` DateTime CODEC(Delta(4), ZSTD(1)),
|
||||
\`End\` DateTime CODEC(Delta(4), ZSTD(1)),
|
||||
INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(Start)
|
||||
ORDER BY (TraceId, Start)
|
||||
TTL toDateTime(Start) + toIntervalHour(730)
|
||||
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
|
||||
"
|
||||
|
||||
clickhouse-client --database="${CLICKHOUSE_DATABASE}" --query "
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv TO otel_traces_trace_id_ts
|
||||
(
|
||||
\`TraceId\` String,
|
||||
\`Start\` DateTime64(9),
|
||||
\`End\` DateTime64(9)
|
||||
)
|
||||
AS SELECT
|
||||
TraceId,
|
||||
min(Timestamp) AS Start,
|
||||
max(Timestamp) AS End
|
||||
FROM otel_traces
|
||||
WHERE TraceId != ''
|
||||
GROUP BY TraceId
|
||||
"
|
||||
|
||||
echo "✅ All 9 OTEL tables created successfully"
|
||||
echo "===================================================================="
|
||||
@@ -0,0 +1,54 @@
|
||||
file_format: '1.0'
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
batch:
|
||||
memory_limiter:
|
||||
# 80% of maximum memory up to 2G
|
||||
limit_mib: 1500
|
||||
# 25% of limit up to 2G
|
||||
spike_limit_mib: 512
|
||||
check_interval: 5s
|
||||
|
||||
exporters:
|
||||
clickhouse:
|
||||
endpoint: tcp://${env:INIT_DB_HOST}:9000?dial_timeout=10s
|
||||
database: ${env:INIT_DB_DATABASE}
|
||||
username: ${env:INIT_DB_USERNAME}
|
||||
password: ${env:INIT_DB_PASSWORD}
|
||||
ttl: 730h
|
||||
logs_table_name: otel_logs
|
||||
traces_table_name: otel_traces
|
||||
# Metrics use separate tables by type: otel_metrics_gauge, otel_metrics_sum,
|
||||
# otel_metrics_histogram, otel_metrics_summary, otel_metrics_exponential_histogram
|
||||
timeout: 5s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 5s
|
||||
max_interval: 30s
|
||||
max_elapsed_time: 300s
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [clickhouse]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [clickhouse]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [clickhouse]
|
||||
# telemetry:
|
||||
# metrics:
|
||||
# address: localhost:8888
|
||||
Reference in New Issue
Block a user