作者：李建生
歸檔：部署文檔

第1章 安裝準備

由于Elasticsearch、Logstash、Kibana均不能以root賬號運行。
但是Linux對非root賬號可并發操作的文件、線程都有限制。
所以，部署ELK相關的機器都要調整：

1.1 安裝jdk

yum install java-1.8.0-openjdk  java-1.8.0-openjdk-devel  -y
[root@elk3 tools]# echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.51.x86_64 \
> export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar\
> export PATH=$PATH:$JAVA_HOME/bin" >> /etc/profile 
[root@elk2 ~]# source  /etc/profile
[root@elk3 tools]# java -version
openjdk version "1.8.0_161"
OpenJDK Runtime Environment (build 1.8.0_161-b14)
OpenJDK 64-Bit Server VM (build 25.161-b14, mixed mode)

1.2 修改文件限制

# 修改系統文件
vim /etc/security/limits.conf
#增加的內容
* soft nofile 65536
* hard nofile 65536
* soft nproc 2048
* hard nproc 4096

1.3 調整進程數

#修改系統文件
vim /etc/security/limits.d/20-nproc.conf
#調整成以下配置
*          soft    nproc     4096
root       soft    nproc     unlimited

1.4 調整虛擬內存&最大并發連接

#修改系統文件
vim /etc/sysctl.conf
#增加的內容
vm.max_map_count=655360
fs.file-max=655360
vm.max_map_count=262144

1.5 創建普通用戶

groupadd elsearch  
useradd elsearch -g elsearch -p 123456
chown -R elsearch.elsearch /server/tools/elasticsearch-6.2.3

1.6 創建日志和數據目錄

mkdir -p /var/data/elasticsearch &&  chown -R elsearch.elsearch /var/data/elasticsearch
mkdir -p /var/log/elasticsearch &&   chown -R elsearch.elsearch /var/log/elasticsearch

第2章 部署ELK

Elasticsearch是一個高度可擴展的開源全文搜索和分析引擎。它允許您快速，近實時地存儲，搜索和分析大量數據。它通常用作支持具有復雜搜索功能和需求的應用程序的底層引擎/技術。

2.1 安裝elasticsearch

2.1.1 配置elasticsearch配置文件

cat >/server/tools/elasticsearch-6.2.3/config/elasticsearch.yml <<END
cluster.name: ES
node.name: ES0
path.data: /var/data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.0.0.19
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["10.0.0.19:9300", "10.0.0.21:9300","10.0.0.22:9300"]
discovery.zen.minimum_master_nodes: 2
END
cat >/server/tools/elasticsearch-6.2.3/config/elasticsearch.yml <<END
cluster.name: ES
node.name: ES1
path.data: /var/data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.0.0.21
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["10.0.0.19:9300", "10.0.0.21:9300","10.0.0.22:9300"]
discovery.zen.minimum_master_nodes: 2
END

cat >/server/tools/elasticsearch-6.2.3/config/elasticsearch.yml <<END
cluster.name: ES
node.name: ES2
path.data: /var/data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.0.0.22
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["10.0.0.19:9300", "10.0.0.21:9300","10.0.0.22:9300"]
discovery.zen.minimum_master_nodes: 2
END

2.1.2 啟動

elsearch@elk1 elasticsearch-6.2.3]$ ./bin/elasticsearch -d #后臺啟動

2.2 安裝配置Logstash

2.2.1 安裝準備

[root@elk3 ~]# mkdir -p /server/tools
[root@elk3 ~]# cd /server/tools/
[root@elk3 tools]# tar xf  kibana-6.2.3-linux-x86_64.tar.gz 
[root@elk3 tools]# tar xf  logstash-6.2.3.tar.gz
groupadd elsearch  
useradd elsearch -g elsearch -p 123456
chown -R elsearch.elsearch /server/tools/

2.2.2 配置Logstash

數據&日志目錄
#創建Logstash數據目錄
#創建Logstash日志目錄
mkdir -p /var/data/logstash &&  chown -R elsearch.elsearch /var/data/logstash
mkdir -p /var/logs/logstash &&   chown -R elsearch.elsearch /var/logs/logstash
配置數據&日志目錄
#打開目錄
cd /server/tools/logstash-6.2.3/
#修改配置
vim config/logstash.yml
#增加以下內容
path.data: /var/data/logstash
path.logs: /var/logs/logstash
配置Redis&Elasticsearch
[root@elk3 config]# cat system.conf 
input {
    file {
        path => "/var/log/*"
        type => "system"
        start_position => "beginning"
    }
}
output {
    elasticsearch {
        hosts => ["10.0.0.21:9200","10.0.0.19:9200","10.0.0.22:9200"]
        index => "system-%{+YYYY.MM.dd}"
    }
}
[root@elk3 config]#

注：該配置就是從redis中讀取數據，然后寫入指定的elasticsearch

2.2.3 啟動logstash

切換普通用戶
[root@elk3 config]# su - elsearch
#進入Logstash根目錄
[elsearch@elk3 ~]$ cd /server/tools/logstash-6.2.3/
#啟動
/bin/logstash -f config/system.conf
啟動成功后，在啟動輸出的最后一行會看到如下信息：
[2018-04-03T10:54:35,819][INFO ][logstash.pipeline        ] Pipeline started succesfully {:pipeline_id=>"main", :thread=>"#<Thread:0x15a6d6d9 run>"}
[2018-04-03T10:54:36,086][INFO ][logstash.agent           ] Pipelines running {:count=>1, :pipelines=>["main"]}

2.3 Kibana 配置

2.3.1 安裝準備

[root@elk3 tools]# tar xf kibana-6.2.3-linux-x86_64
chown elsearch:elsearch /server/tools/kibana-6.2.3-linux-x86_64

2.3.2 修改配置

#進入kibana根目錄
cd /server/tools/kibana-6.2.3-linux-x86_64
#修改配置
vi config/kibana.yml
#修改以下內容
[root@elk3 kibana-6.2.3-linux-x86_64]# egrep -v "^#|^$" config/kibana.yml 
server.port: 5601
server.host: "10.0.0.24"
elasticsearch.url: "http://10.0.0.21:9200"
[root@elk3 kibana-6.2.3-linux-x86_64]#

2.3.3 啟動

切換普通用戶啟動
[root@elk3 kibana-6.2.3-linux-x86_64]# su - elsearch 
 [elsearch@elk3 ~]$ cd /server/tools/kibana-6.2.3-linux-x86_64/
[elsearch@elk3 kibana-6.2.3-linux-x86_64]$ bin/kibana

2.3.4 瀏覽器訪問

http://10.0.0.24:5601

第3章 Filebeat配置

系統Filebeat模塊收集并分析由基于Unix / Linux的常見發行版的系統日志記錄服務創建的日志。

3.1 安裝

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.2.3-x86_64.rpm
rpm -vi filebeat-6.2.3-x86_64.rpm
安裝插件
bin/elasticsearch-plugin install ingest-geoip
bin/elasticsearch-plugin install ingest-user-agent

3.2 修改/etc/filebeat/filebeat.yml以設置連接信息

output.elasticsearch:
  hosts: ["<es_url>"]
  username: "elastic"
  password: "<password>"
setup.kibana:
  host: "<kibana_url>"

3.3 啟用并配置系統模塊

filebeat modules enable system
修改/etc/filebeat/modules.d/system.yml文件中的設置。

3.4 啟動Filebeat

filebeat setup
systemctl restart filebeat.service

3.5 配置nginx和mysql日志查詢

#vim /etc/filebeat/modules.d/mysql.yml
- module: mysql
  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/data/mysql/czypweb.err*"]

  # Slow logs
  slowlog:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
var.paths: ["/data/mysql/mysql-slow.log*"]
#vim /etc/filebeat/modules.d/nginx.yml

- module: nginx
  # Access logs
  access:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/usr/local/openresty/nginx/logs/access.log*"]

  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/usr/local/openresty/nginx/logs/error.log*"]
修改filebeat配置文件開啟模塊功能

第4章 System metrics

系統Metricbeat模塊從主機收集CPU，內存，網絡和磁盤統計信息。它收集系統范圍的統計信息以及每個進程和每個文件系統的統計信息。

4.1 下載

curl -L -O https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-6.2.3-x86_64.rpm
sudo rpm -vi metricbeat-6.2.3-x86_64.rpm

4.2 配置

/etc/metricbeat/metricbeat.yml
metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
  reload.period: 10s
output.elasticsearch:
  hosts: ["<es_url>"]
  username: "elastic"
  password: "<password>"
setup.kibana:
  host: "<kibana_url>"

4.3 啟用并配置系統模塊

metricbeat modules enable system

4.4 啟動Metricbeat

metricbeat setup
service metricbeat start
systemctl enable metricbeat.==service開機自啟==