基于FlumeNG+Kafka+ElasticSearch+Kibana的日志系统

系统"/>

基于FlumeNG+Kafka+ElasticSearch+Kibana的日志系统

为什么80%的码农都做不了架构师？>>>   

环境准备

1.服务器概览

2.服务器环境安装

三台服务均安装jdk1.8(此处省略，自行安装)。

node1.fek安装nginx，模仿需要采集日志的web容器(此处省略，自行安装)。

三台服务均执行

#添加host
192.168.2.161 node1.fek
192.168.2.162 node2.fek
192.168.2.163 master.fek192.168.2.151 master.kafka
192.168.2.152 worker1.kafka
192.168.2.153 worker2.kafka#执行以下命令关闭防火墙
[root@node1 ~]systemctl stop firewalld && systemctl disable firewalld
[root@node1 ~]setenforce 0#将SELINUX的值改成disabled
[root@node1 ~]vim /etc/selinux/configSELINUX=disabled#修改系统参数
sudo vim /etc/security/limits.conf
*               soft    nproc           65536
*               hard    nproc           65536
*               soft    nofile          65536
*               hard    nofile          65536
sudo vim /etc/sysctl.conf
vm.max_map_count= 262144
sudo sysctl -p#重启服务器
[root@node1 ~]reboot

elasticsearch集群安装

注意：安装elasticsearch需要非root用户，此处我们用user用户

mkdir -p /opt/elasticsearch
chown user:user /opt/elasticsearch
su user

下载

wget .3.0.tar.gztar -zxvf elasticsearch-6.3.0.tar.gzcd elasticsearch-6.3.0/configmkdir -p /opt/elasticsearch/data
mkdir -p /opt/elasticsearch/logs

修改配置参数 elasticsearch.yml

cluster.name: fek-cluster                                 # 集群名称
node.name: els1                                             # 节点名称，仅仅是描述名称，用于在日志中区分path.data: /opt/elasticsearch/data                           # 数据的默认存放路径
path.logs: /opt/elasticsearch/logs                           # 日志的默认存放路径network.host: 192.168.2.163                                   # 当前节点的IP地址
http.port: 9200                                             # 对外提供服务的端口，9300为集群服务的端口discovery.zen.ping.unicast.hosts: ["node1.fek", "node2.fek", "master.fek"]       
# 集群个节点IP地址，也可以使用els、els.shuaiguoxia等名称，需要各节点能够解析discovery.zen.minimum_master_nodes: 2                       # 为了避免脑裂，集群节点数最少为 半数+1

jvm.options

-Xms1g                                                  # JVM最大、最小使用内存
-Xmx1g

对于node1.fek node2.fek,安装实际情况修改elasticsearch.yml参数

启动集群

./bin/elasticsearch -d

访问http://192.168.2.163:9200/_cluster/health?pretty，查看集群状态

{"cluster_name" : "fek-cluster","status" : "green","timed_out" : false,"number_of_nodes" : 3,"number_of_data_nodes" : 3,"active_primary_shards" : 0,"active_shards" : 0,"relocating_shards" : 0,"initializing_shards" : 0,"unassigned_shards" : 0,"delayed_unassigned_shards" : 0,"number_of_pending_tasks" : 0,"number_of_in_flight_fetch" : 0,"task_max_waiting_in_queue_millis" : 0,"active_shards_percent_as_number" : 100.0
}

http://192.168.2.163:9200/_cluster/state/nodes?pretty

{"cluster_name" : "fek-cluster","compressed_size_in_bytes" : 9419,"nodes" : {"NaRMw2usS0q28ZscXMHHcQ" : {"name" : "node2.fek","ephemeral_id" : "1BF5Tiw_RbCq9FsbwuyWjA","transport_address" : "192.168.2.162:9300","attributes" : {"ml.machine_memory" : "1021931520","ml.max_open_jobs" : "20","xpack.installed" : "true","ml.enabled" : "true"}},"y3UhYgopT12alHMlJDJlWQ" : {"name" : "node1.fek","ephemeral_id" : "j5K2Re4QSW-GcJMVVXPP5g","transport_address" : "192.168.2.161:9300","attributes" : {"ml.machine_memory" : "1021931520","ml.max_open_jobs" : "20","xpack.installed" : "true","ml.enabled" : "true"}},"kO7CFyN3RKWLURRtrhwTMQ" : {"name" : "master.fek","ephemeral_id" : "JH2omxVpRVyqQN47RMENsQ","transport_address" : "192.168.2.163:9300","attributes" : {"ml.machine_memory" : "1021931520","ml.max_open_jobs" : "20","xpack.installed" : "true","ml.enabled" : "true"}}}
}

Kafka集群安装

Kafka集群部署使用：

FlumeNG安装

***0.消费Kafka信息，传入ES

mkdir -p /opt/app/
cd /opt/app/
wget .0-SNAPSHOT.jar
java -jar kafka-es-1.0-SNAPSHOT.jar

源码地址：kafka-es 可以根据项目需要，过滤清洗日志，在入ES的库时可以异步写入HDFS，做大数据分析

此处选择Kafka是因为其削峰能力。

1. 配置FlumeNG 此处，我们使用node1.fek上nginx模拟需要采集日志的应用，nginx日志位于/usr/local/nginx/logs目录

mkdir -p /opt/flume/
cd /opt/flume/
wget .8.0/apache-flume-1.8.0-bin.tar.gz
tar -zxvf apache-flume-1.8.0-bin.tar.gz

配置flume-fek.conf，可以配置Spooling Directory Source（监听一个目录）或 Exec Source（监听一个文件）

#Spooling Directory Source
agent.sources = sc_nginx
agent.channels = cl_nginx
agent.sinks = sk_nginxagent.sources.sc_nginx.type = spooldir
agent.sources.sc_nginx.spoolDir = /usr/local/nginx/logs
agent.sources.sc_nginx.fileHeader = trueagent.channels.cl_nginx.type = memory
agent.channels.cl_nginx.capacity = 1000
agent.channels.cl_nginx.transactionCapacity = 100agent1.sinks.sk_nginx.type = org.apache.flume.sink.kafka.KafkaSink
agent1.sinks.sk_nginx.kafka.topic = fek_topic
agent1.sinks.sk_nginx.kafka.bootstrap.servers = 192.168.2.151:9092,192.168.2.152:9092,192.168.2.153:9092
agent1.sinks.sk_nginx.kafka.flumeBatchSize = 20
agent1.sinks.sk_nginx.kafka.producer.acks = 1
agent1.sinks.sk_nginx.kafka.producer.linger.ms = 1
agent1.sinks.sk_nginx.kafka.producerpression.type = snappyagent.sources.sc_nginx.channels = cl_nginx
agent.sinks.sk_nginx.channel = cl_nginx

或

#Exec Source
agent.sources = sc_nginx
agent.channels = cl_nginx
agent.sinks = sk_nginxagent.sources.sc_nginx.type = exec
agent.sources.sc_nginxmand = tail -F /usr/local/nginx/logs/access.logagent.channels.cl_nginx.type = memory
agent.channels.cl_nginx.capacity = 1000
agent.channels.cl_nginx.transactionCapacity = 100agent1.sinks.sk_nginx.type = org.apache.flume.sink.kafka.KafkaSink
agent1.sinks.sk_nginx.kafka.topic = fek_topic
agent1.sinks.sk_nginx.kafka.bootstrap.servers = 192.168.2.151:9092,192.168.2.152:9092,192.168.2.153:9092
agent1.sinks.sk_nginx.kafka.flumeBatchSize = 20
agent1.sinks.sk_nginx.kafka.producer.acks = 1
agent1.sinks.sk_nginx.kafka.producer.linger.ms = 1
agent1.sinks.sk_nginx.kafka.producerpression.type = snappyagent.sources.sc_nginx.channels = cl_nginx
agent.sinks.sk_nginx.channel = cl_nginx

配置文件说明:

2. 启动FlumeNG

cd /opt/flume/apache-flume-1.8.0/
mkdir -p /opt/flume/apache-flume-1.8.0/logs
bin/flume-ng agent --conf ./conf/ -f conf/flume-fek.conf -n agent > ./logs/start.log 2>&1 &

启动命令参数说明：

Kibana安装

在master.fek上安装Kibana

mkdir -p /opt/kibana/
cd /opt/kibana/
wget .3.0-linux-x86_64.tar.gz
tar -zxvf kibana-6.3.0-linux-x86_64.tar.gz
mv kibana-6.3.0-linux-x86_64 kibana-6.3.0

编辑kibana-6.3.0/config/kibana.yml

#配置本机ip  
server.host: "master.fek"
#配置es集群url  
elasticsearch.url: "http://192.168.2.163:9200"

启动

cd kibana-6.3.0//bin./kibana &

访问：http://192.168.2.163:5061

转载于: