• 生活的道路一旦选定,就要勇敢地走到底,决不回头。——左拉
  • 坚强的信心,能使平凡的人做出惊人的事业。——马尔顿
  • 人不可有傲气,但不可无傲骨。 --徐悲鸿
  • 古之立大志者,不惟有超世之才,亦必有坚韧不拔之志。 --苏轼
  • 时间像海绵里的水,只要你愿意挤,总还是有的。 --鲁迅

kafka+ELKF集群搭建

DevOps zkinogg 2年前 (2020-08-18) 614次浏览 0个评论

ELK+Kafka集群搭建

环境准备

主机名 ip 端口 服务名
m01 10.0.0.61 ansible服务端
db01 10.0.0.51 2181.9092 zookeeper,kafka
db02 10.0.0.52 2181.9092.2888 zookeeper,kafka
db03 10.0.0.53 2181.9092.3888 zookeeper,kafka
db04 10.0.0.54 9200.9300.5601 elasticsearch,kibana
db05 10.0.0.55 9200.9300.9600 elasticsearch,logstash
db06 10.0.0.56 9200.9300 elasticsearch
web01 10.0.0.7 80.8080 filebeat,nginx,tomcat

架构思路图

duomwD.png

ansible搭建ELK并测试

这里一定记得内存调2G!配置时间同步 !!

具体步骤

# 0.发送密钥
[root@m01 ~]# ssh-keygen
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.54
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.55
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.56
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.7

# 1.准备elasticsearch的roles目录
[root@m01 ~]# mkdir ansible_elasticsearch
[root@m01 ~]# cd ansible_elasticsearch/
[root@m01 ansible_elasticsearch]# ansible-galaxy init elasticsearch

# 2.准备hosts文件
[root@m01 ansible_elasticsearch]# cat hosts 
[db_group]
db04 ansible_ssh_host=172.16.1.54
db05 ansible_ssh_host=172.16.1.55
db06 ansible_ssh_host=172.16.1.56
[web_group]
web01 ansible_ssh_host=172.16.1.7

# 3.准备site.yml文件
[root@m01 ansible_elasticsearch]# cat site.yml 
- hosts: all
  roles:
    - { role: elasticsearch , when: (ansible_fqdn is match 'db*') or (ansible_fqdn is match 'web*')}
    
# 4.用jinjia模板准备elasticsearch的配置文件
[root@m01 elasticsearch]# cat templates/elasticsearch.yml.j2 
cluster.name: es-cluster
path.data: /service/es/data
path.logs: /service/es/logs
bootstrap.memory_lock: true
{% if ansible_fqdn == 'db04' %}
node.name: node-1
network.host: 10.0.0.54,127.0.0.1
{% elif ansible_fqdn == 'db05' %}
node.name: node-2
network.host: 10.0.0.55,127.0.0.1
{% else %}
node.name: node-3
network.host: 10.0.0.56,127.0.0.1
{% endif %}
http.port: 9200
discovery.zen.ping.unicast.hosts: ["10.0.0.54", "10.0.0.55","10.0.0.56"]
discovery.zen.minimum_master_nodes: 2


# 5.准备文件
[root@m01 files]# ll
total 635780
-rw-r--r-- 1 root root  11100954 Aug 14 09:13 apache-tomcat-10.0.0-M7.tar.gz # tomcat包
-rw-r--r-- 1 root root 114059630 Aug  9 18:54 elasticsearch-6.6.0.rpm # Es安装包
-rw-r--r-- 1 root root      1703 Aug 11 15:17 elasticsearch.service  # es启动脚本
-rw-r--r-- 1 root root 170023183 Aug  9 18:54 jdk-8u181-linux-x64.rpm  # java环境包
-rw-r--r-- 1 root root