Index: web-module/script/yml/monitoring/allinone.yml =================================================================== diff -u -rdc34710585997ad951a60b570f7e635ba2ae02f1 -r48c8cb6664d88be92225d57ef2d9b8a08e1573a2 --- web-module/script/yml/monitoring/allinone.yml (.../allinone.yml) (revision dc34710585997ad951a60b570f7e635ba2ae02f1) +++ web-module/script/yml/monitoring/allinone.yml (.../allinone.yml) (revision 48c8cb6664d88be92225d57ef2d9b8a08e1573a2) @@ -24,7 +24,12 @@ image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0 hostname: es-master-01 volumes: - - es-master-01-data:/usr/share/elasticsearch/data + #- es-master-01-data:/usr/share/elasticsearch/data + - type: volume + source: es-master-01-data + target: /usr/share/elasticsearch/data + volume: + nocopy: true - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro environment: &es-master-env @@ -51,13 +56,25 @@ endpoint_mode: dnsrr mode: replicated replicas: 1 + logging: + driver: "fluentd" + options: + fluentd-address: 192.168.25.46:24224 + fluentd-async-connect: "true" + fluentd-retry-wait: "1s" + fluentd-max-retries: "30" # elasticsearch master node es-master-02: image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0 hostname: es-master-02 volumes: - - es-master-02-data:/usr/share/elasticsearch/data + #- es-master-02-data:/usr/share/elasticsearch/data + - type: volume + source: es-master-02-data + target: /usr/share/elasticsearch/data + volume: + nocopy: true - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro environment: @@ -86,7 +103,12 @@ image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0 hostname: es-master-03 volumes: - - es-master-03-data:/usr/share/elasticsearch/data + #- es-master-03-data:/usr/share/elasticsearch/data + - type: volume + source: es-master-03-data + target: /usr/share/elasticsearch/data + volume: + nocopy: true - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro environment: &es-master-env @@ -165,6 +187,7 @@ ports: - "5000:5000" - "9600:9600" + - "5044:5044" configs: - source: logstash_config target: /usr/share/logstash/config/logstash.yml @@ -179,6 +202,26 @@ mode: replicated replicas: 1 + fluentd: + image: 313devgrp/fluentd:3.1.3 + networks: *efk-network + depends_on: + - es-coordinating + configs: + - source: fluent_conf + target: /fluentd/etc/fluentd.conf + ports: + - "24224:24224" + - "24224:24224/udp" + deploy: + mode: replicated + replicas: 1 + logging: + driver: "json-file" + options: + max-size: "100M" + max-file: "50" + #https://gist.github.com/dkurzaj/2a899de8cb5ae698919f0a9bbf7685f0 zookeeper1: image: wurstmeister/zookeeper:3.4.6 @@ -376,6 +419,8 @@ - 6100:6100/udp configs: + fluent_conf: + file: ./allinone/fluentd/conf/fluent.conf apmserver.conf: file: ./allinone/apm-server.yml influxdb.conf: Index: web-module/script/yml/monitoring/allinone/apm-server.yml =================================================================== diff -u -r3e2c8b7634389a7bc0ddee58812b44d1e711aecc -r48c8cb6664d88be92225d57ef2d9b8a08e1573a2 --- web-module/script/yml/monitoring/allinone/apm-server.yml (.../apm-server.yml) (revision 3e2c8b7634389a7bc0ddee58812b44d1e711aecc) +++ web-module/script/yml/monitoring/allinone/apm-server.yml (.../apm-server.yml) (revision 48c8cb6664d88be92225d57ef2d9b8a08e1573a2) @@ -328,10 +328,10 @@ # Scheme and port can be left out and will be set to the default (http and 5601) # In case you specify and additional path, the scheme is required: http://localhost:5601/path # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 - #host: "localhost:5601" + # host: "kibana:5601" # Optional protocol and basic auth credentials. - #protocol: "https" + #protocol: "http" #username: "elastic" #password: "changeme" @@ -395,7 +395,7 @@ # Scheme and port can be left out and will be set to the default (http and 9200) # In case you specify and additional path, the scheme is required: http://localhost:9200/path # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 - #hosts: ["localhost:9200"] + #hosts: ["es-coordinating:9200"] # Boolean flag to enable or disable the output module. #enabled: true Index: web-module/script/yml/monitoring/allinone/fluentd/Dockerfile =================================================================== diff -u --- web-module/script/yml/monitoring/allinone/fluentd/Dockerfile (revision 0) +++ web-module/script/yml/monitoring/allinone/fluentd/Dockerfile (revision 48c8cb6664d88be92225d57ef2d9b8a08e1573a2) @@ -0,0 +1,4 @@ +FROM fluent/fluentd:v1.11-2 +USER root +RUN ["gem", "install", "fluent-plugin-elasticsearch", "--no-document", "--version", "4.1.4"] +USER fluent Index: web-module/script/yml/monitoring/allinone/fluentd/conf/fluent.conf =================================================================== diff -u --- web-module/script/yml/monitoring/allinone/fluentd/conf/fluent.conf (revision 0) +++ web-module/script/yml/monitoring/allinone/fluentd/conf/fluent.conf (revision 48c8cb6664d88be92225d57ef2d9b8a08e1573a2) @@ -0,0 +1,23 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + @type copy + + @type elasticsearch + host es-coordinating + port 9200 + logstash_format true + logstash_prefix fluentd + logstash_dateformat %Y%m%d + include_tag_key true + type_name access_log + tag_key @log_name + flush_interval 1s + + + @type stdout + + Index: web-module/script/yml/monitoring/allinone/logstash.conf =================================================================== diff -u -r1e186118efb1dc91159fccd5c3aca7478ca78712 -r48c8cb6664d88be92225d57ef2d9b8a08e1573a2 --- web-module/script/yml/monitoring/allinone/logstash.conf (.../logstash.conf) (revision 1e186118efb1dc91159fccd5c3aca7478ca78712) +++ web-module/script/yml/monitoring/allinone/logstash.conf (.../logstash.conf) (revision 48c8cb6664d88be92225d57ef2d9b8a08e1573a2) @@ -37,7 +37,7 @@ elasticsearch { hosts => ["es-coordinating:9200"] - index => "%{[@metadata][beat]}" + index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" document_type => "%{[@metadata][type]}" }