feat: add more services

src/logstash/docker-compose.yaml

@@ -0,0 +1,59 @@
+x-default: &default
+  restart: unless-stopped
+  volumes:
+    - &localtime /etc/localtime:/etc/localtime:ro
+    - &timezone /etc/timezone:/etc/timezone:ro
+  logging:
+    driver: json-file
+    options:
+      max-size: 100m
+
+services:
+  logstash:
+    <<: *default
+    image: docker.elastic.co/logstash/logstash:${LOGSTASH_VERSION:-8.16.1}
+    container_name: logstash
+    ports:
+      - "${LOGSTASH_BEATS_PORT_OVERRIDE:-5044}:5044"
+      - "${LOGSTASH_TCP_PORT_OVERRIDE:-5000}:5000/tcp"
+      - "${LOGSTASH_UDP_PORT_OVERRIDE:-5000}:5000/udp"
+      - "${LOGSTASH_HTTP_PORT_OVERRIDE:-9600}:9600"
+    volumes:
+      - *localtime
+      - *timezone
+      - logstash_data:/usr/share/logstash/data
+      - logstash_logs:/usr/share/logstash/logs
+      - ./pipeline:/usr/share/logstash/pipeline:ro
+
+      # Custom configuration
+      # - ./logstash.yml:/usr/share/logstash/config/logstash.yml:ro
+      # - ./pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro
+    environment:
+      - XPACK_MONITORING_ENABLED=${LOGSTASH_MONITORING_ENABLED:-false}
+      - XPACK_MONITORING_ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOSTS:-http://elasticsearch:9200}
+      - ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOSTS:-http://elasticsearch:9200}
+      - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME:-}
+      - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-}
+      - LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx1g -Xms1g}
+      - PIPELINE_WORKERS=${LOGSTASH_PIPELINE_WORKERS:-2}
+      - PIPELINE_BATCH_SIZE=${LOGSTASH_PIPELINE_BATCH_SIZE:-125}
+      - PIPELINE_BATCH_DELAY=${LOGSTASH_PIPELINE_BATCH_DELAY:-50}
+      - LOG_LEVEL=${LOGSTASH_LOG_LEVEL:-info}
+    deploy:
+      resources:
+        limits:
+          cpus: '1.50'
+          memory: 2G
+        reservations:
+          cpus: '0.50'
+          memory: 1G
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:9600/_node/stats || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
+volumes:
+  logstash_data:
+  logstash_logs:

src/logstash/pipeline/logstash.conf

@@ -0,0 +1,46 @@
+input {
+  beats {
+    port => 5044
+  }
+
+  tcp {
+    port => 5000
+    codec => json_lines
+  }
+
+  udp {
+    port => 5000
+    codec => json_lines
+  }
+}
+
+filter {
+  if [fields][log_type] == "syslog" {
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} %{GREEDYDATA:message}" }
+    }
+  }
+
+  if [fields][log_type] == "apache" {
+    grok {
+      match => { "message" => "%{COMBINEDAPACHELOG}" }
+    }
+  }
+
+  date {
+    match => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}"]
+    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+    user => "${ELASTICSEARCH_USERNAME:}"
+    password => "${ELASTICSEARCH_PASSWORD:}"
+  }
+
+  stdout {
+    codec => rubydebug
+  }
+}