ELK容器部署运行

【历史库存文档】用容器方式运行，将elk的环境使用容器编排的方式来实现。过程如下：

filebeat =>> redis =>> logstash =>> elasticsearch =>> kibana

服务编排目录结构：

tree ./
./
├── docker-compose.yml
├── elasticsearch
│   └── elasticsearch.yml
├── logstash
│   └── logstash.conf
└── redis
    └── redis.conf

docker-compose.yml 内容：

version: '3'

services:

  elasticsearch:
    image: elasticsearch:6.4.2
    container_name: elasticsearch
    hostname: elasticsearch

    ports:
      - "9200:9200"
      - "9300:9300"

    volumes:
      - ./elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    environment:
      ES_JAVA_OPTS: "-Xmx1024m -Xms1024m"
    networks:
      - elk

  logstash:
    image: logstash:6.4.2
    container_name: logstash
    hostname: logstash
    command: logstash -f /etc/logstash/conf.d/logstash.conf
    volumes:
      - ./logstash:/etc/logstash/conf.d
    ports:
      - "5000:5000"
    networks:
      - elk


  kibana:
    image: kibana:6.4.2
    container_name: kibana
    hostname: kibana
    environment: #在kibana中指定es地址，使用容器名称进行连接
      - ELASTICSEARCH_URL=http://elasticsearch:9200
    ports:
      - "5601:5601"
    networks:
      - elk

  redis:
    container_name: redis
    hostname: redis
    image: redis
    ports:
      - "6379:6379"
    volumes:
#      - /data/redis:/data/redis
      - ./redis/redis.conf:/opt/server/redis/redis.conf
    command: redis-server /opt/server/redis/redis.conf

    networks:
      - elk


networks:
  elk:
    driver: bridge

elasticsearch.yml 内容：

cluster.name: "docker-cluster"
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
discovery.type: single-node

logstash.conf 内容：

input {

        redis {

        host => "192.168.2.134"
        port => "6379"
        key  => "filebeat"
        data_type => "list"
        password => "123456789"
        threads   => "5"


}

}

filter {

        date {
        match => [ "timestamp","dd/MMM/YYYY:H:m:s Z" ]
        remove_field => "timestamp"
        }


}

output {

        elasticsearch {

                hosts => "elasticsearch:9200"
                index => "logstash-%{+YYYY.MM.dd}"
                document_type => "java_logs"
        }

}

redis.conf 内容：redis配置只修改几个选项，其他保持了默认，修改内容如下：

bind 0.0.0.0             #可连接地址段，这里设置允许所有
requirepass 123456789  #密码

远程数据源节点日志收集工具部署使用filebeat。同样采用容器来运行。

filebeat的部署内容如下：

version: '3'

services:

  filebeat:
    image: elastic/filebeat:6.4.2
    container_name: filebeat
    hostname: filebeat
    privileged: true
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /logs/app:/logs/app

filebeat.yml 内容如下：

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - "/logs/app/*.log"
  document_type: "java_logs"
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
#  tail_files: true   #如果不收集以前的历史日志，这里要启用该选项，只收集最新的日志。

output.redis:
  hosts: ["192.168.2.134:6379"]
  port: 6379
  password: "123456789"   #这里密码要和redis中设置的一样，否则连不上
  key: "filebeat"
  db: 0
  timeout: 5