• 生活的道路一旦选定,就要勇敢地走到底,决不回头。——左拉
  • 坚强的信心,能使平凡的人做出惊人的事业。——马尔顿
  • 人不可有傲气,但不可无傲骨。 --徐悲鸿
  • 古之立大志者,不惟有超世之才,亦必有坚韧不拔之志。 --苏轼
  • 时间像海绵里的水,只要你愿意挤,总还是有的。 --鲁迅

ansible实现ELK

DevOps zkinogg 1年前 (2020-08-14) 463次浏览 0个评论

ansible实现ELK

一.环境准备

内存调2G!配置时间同步 !!!

主机名 ip 端口 服务
m01 10.0.0.61 22 ansible
db04 10.0.0.54 5601.9200.9300 kibana,elasticsearch
db05 10.0.0.55 9200.9300 kibana,elasticsearch
db06 10.0.0.56 9200.9300 kibana,elasticsearch
web01 10.0.0.7 8080.80 nginx,tomcat,logstash

二.操作步骤

# 0.发送密钥
[root@m01 ~]# ssh-keygen
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.54
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.55
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.56
[root@m01 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@172.16.1.7

# 1.准备elasticsearch的roles目录
[root@m01 ~]# mkdir ansible_elasticsearch
[root@m01 ~]# cd ansible_elasticsearch/
[root@m01 ansible_elasticsearch]# ansible-galaxy init elasticsearch

# 2.准备hosts文件
[root@m01 ansible_elasticsearch]# cat hosts 
[db_group]
db04 ansible_ssh_host=172.16.1.54
db05 ansible_ssh_host=172.16.1.55
db06 ansible_ssh_host=172.16.1.56
[web_group]
web01 ansible_ssh_host=172.16.1.7

# 3.准备site.yml文件
[root@m01 ansible_elasticsearch]# cat site.yml 
- hosts: all
  roles:
    - { role: elasticsearch , when: (ansible_fqdn is match 'db*') or (ansible_fqdn is match 'web*')}
    
# 4.用jinjia模板准备elasticsearch的配置文件
[root@m01 elasticsearch]# cat templates/elasticsearch.yml.j2 
cluster.name: es-cluster
path.data: /service/es/data
path.logs: /service/es/logs
bootstrap.memory_lock: true
{% if ansible_fqdn == 'db04' %}
node.name: node-1
network.host: 10.0.0.54,127.0.0.1
{% elif ansible_fqdn == 'db05' %}
node.name: node-2
network.host: 10.0.0.55,127.0.0.1
{% else %}
node.name: node-3
network.host: 10.0.0.56,127.0.0.1
{% endif %}
http.port: 9200
discovery.zen.ping.unicast.hosts: ["10.0.0.54", "10.0.0.55","10.0.0.56"]
discovery.zen.minimum_master_nodes: 2


# 5.准备文件
[root@m01 files]# ll
total 635780
-rw-r--r-- 1 root root  11100954 Aug 14 09:13 apache-tomcat-10.0.0-M7.tar.gz # tomcat包
-rw-r--r-- 1 root root 114059630 Aug  9 18:54 elasticsearch-6.6.0.rpm # Es安装包
-rw-r--r-- 1 root root      1703 Aug 11 15:17 elasticsearch.service  # es启动脚本
-rw-r--r-- 1 root root 170023183 Aug  9 18:54 jdk-8u181-linux-x64.rpm  # java环境包
-rw-r--r-- 1 root root 185123116 Aug  9 18:54 kibana-6.6.0-x86_64.rpm  # kibana安装包
-rw-r--r-- 1 root root       190 Aug 11 15:47 kibana.yml            # kibana配置文件
-rw-r--r-- 1 root root 170703770 Aug  9 18:54 logstash-6.6.0.rpm    # logstash安装包
-rw-r--r-- 1 root root        43 Aug 14 22:32 logstash.sh           # logstash环境变量文件
-rw-r--r-- 1 root root       366 Aug 15 01:46 nginx_tomcat.conf     # logstash收集配置文件

##  kibana配置文件
[root@m01 elasticsearch]# cat files/kibana.yml 
#进程的端口
server.port: 5601
#监听地址
server.host: "10.0.0.54"
#指定ES的地址
elasticsearch.hosts: ["http://10.0.0.54:9200"]
#kibana也会创建索引
kibana.index: ".kibana"


## Es进程启动文件
[root@m01 elasticsearch]# cat files/elasticsearch.service 
[Unit]
Description=Elasticsearch
Documentation=http://www.elastic.co
Wants=network-online.target
After=network-online.target

[Service]
RuntimeDirectory=elasticsearch
PrivateTmp=true
Environment=ES_HOME=/usr/share/elasticsearch
Environment=ES_PATH_CONF=/etc/elasticsearch
Environment=PID_DIR=/var/run/elasticsearch
EnvironmentFile=-/etc/sysconfig/elasticsearch
LimitMEMLOCK=infinity

WorkingDirectory=/usr/share/elasticsearch

User=elasticsearch
Group=elasticsearch

ExecStart=/usr/share/elasticsearch/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid --quiet

# StandardOutput is configured to redirect to journalctl since
# some error messages may be logged in standard output before
# elasticsearch logging system is initialized. Elasticsearch
# stores its logs in /var/log/elasticsearch and does not use
# journalctl by default. If you also want to enable journalctl
# logging, you can simply remove the "quiet" option from ExecStart.
StandardOutput=journal
StandardError=inherit

# Specifies the maximum file descriptor number that can be opened by this process
LimitNOFILE=65536

# Specifies the maximum number of processes
LimitNPROC=4096

# Specifies the maximum size of virtual memory
LimitAS=infinity

# Specifies the maximum file size
LimitFSIZE=infinity

# Disable timeout logic and wait until process is stopped
TimeoutStopSec=0

# SIGTERM signal is used to stop the Java process
KillSignal=SIGTERM

# Send the signal only to the JVM rather than its control group
KillMode=process

# Java process is never killed
SendSIGKILL=no

# When a JVM receives a SIGTERM signal it exits with code 143
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target

# Built for packages-6.6.0 (packages)


##  logstash收集配置文件
[root@m01 files]# cat nginx_tomcat.conf 
input {
  file {
    type => "tomcat_log"
    path => "/usr/local/tomcat/logs/localhost_access_log.*.txt"
    start_position => "beginning"
  }
  file {
    type => "nginx_log"
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
  }
}
output {
    elasticsearch {
      hosts => ["10.0.0.54:9200"]
      index => "%{type}_%{+YYYY-MM-dd}"
  }
}


## logstash环境变量文件
[root@m01 files]# cat logstash.sh 
export PATH=/usr/share/logstash/bin/:$PATH


## 7.编写ELK的tasks
[root@m01 tasks]# ll
total 20
-rw-r--r-- 1 root root  147 Aug 14 23:03 main.yml
-rw-r--r-- 1 root root 2071 Aug 15 01:45 Yum_ES.yml  # 安装配置并启动ES
-rw-r--r-- 1 root root  503 Aug 11 16:19 Yum_kibana.yml  # 安装配置并启动kibana
-rw-r--r-- 1 root root  935 Aug 15 00:56 Yum_logstash.yml  # 安装配置并启动logstash
-rw-r--r-- 1 root root  621 Aug 15 00:26 Yum_Nginx_Tomcat.yml  # 安装配置并启动nginx和tomcat

[root@m01 tasks]# cat main.yml 
---
# tasks file for elasticsearch
- include : Yum_ES.yml          
- include : Yum_kibana.yml
- include : Yum_Nginx_Tomcat.yml
- include : Yum_logstash.yml

[root@m01 tasks]# cat Yum_ES.yml 
- name: Yum ntpdate                 # 安装时间同步服务
  yum:
    name: ntpdate
    state: present
  when: (ansible_fqdn is match 'db*') or (ansible_fqdn is match 'web*')

- name: ntpdate aliyun time         # 同步阿里云时间
  shell: 'ntpdate time1.aliyun.com'
  when: (ansible_fqdn is match 'db*') or (ansible_fqdn is match 'web*')

- name: Push java rpm               # 推java环境包
  copy:
    src: jdk-8u181-linux-x64.rpm
    dest: /root/jdk-8u181-linux-x64.rpm
  when: (ansible_fqdn is match 'db*') or (ansible_fqdn is match 'web*')

- name: Yum java rpm                # 安装java环境包
  yum:
   name: /root/jdk-8u181-linux-x64.rpm
   state: present
  when: (ansible_fqdn is match 'db*') or (ansible_fqdn is match 'web*')

- name: Source Profile              # 刷新环境变量
  shell: 'source /etc/profile'
  when: (ansible_fqdn is match 'db*') or (ansible_fqdn is match 'web*')

- name: Push elasticsearch rpm      # 推Es安装包
  copy:
    src: elasticsearch-6.6.0.rpm
    dest: /root/elasticsearch-6.6.0.rpm
  when: ansible_fqdn is match 'db*'

- name: Yum elasticsearch rpm       # 安装Es
  yum:
    name: /root/elasticsearch-6.6.0.rpm
    state: present
  when: ansible_fqdn is match 'db*'

- name: reload systemd              # 根据提示重启systemcd
  shell: 'systemctl daemon-reload'
  when: ansible_fqdn is match 'db*'

- name: service start               # 根据提示启动Es并添加开机自启
  service:
     name: elasticsearch.service
     state: started
     enabled: yes
  when: ansible_fqdn is match 'db*'

- name: Push elasticsearch config       # 推Es配置文件
  template:
    src: elasticsearch.yml.j2
    dest: /etc/elasticsearch/elasticsearch.yml
  when: ansible_fqdn is match 'db*'

- name: mkdir config directory          # 根据配置文件创建目录并授权
  file:
    path: /service/es/{{ item }}
    owner: elasticsearch
    group: elasticsearch
    state: directory
    recurse: yes
  with_items: 
    - data
    - logs
  when: ansible_fqdn is match 'db*'

- name: Push systemd start config       # 推systemd启动文件
  copy:
    src: elasticsearch.service
    dest: /usr/lib/systemd/system/elasticsearch.service
    mode: 0755
  when: ansible_fqdn is match 'db*'

- name: reload systemd                  # 重启systemd
  shell: 'systemctl daemon-reload'
  when: ansible_fqdn is match 'db*'

- name: service start                       # 开启Es
  service: 
     name: elasticsearch.service
     state: restarted
     enabled: yes
  when: ansible_fqdn is match 'db*'




[root@m01 tasks]# cat Yum_kibana.yml 
- name: Push kibana rpm                     # 推kibana安装 包
  copy:
    src: kibana-6.6.0-x86_64.rpm
    dest: /root/kibana-6.6.0-x86_64.rpm
  when: ansible_fqdn == 'db04'

- name: Yum kibana rpm                      # 安装kibana
  yum:
    name: /root/kibana-6.6.0-x86_64.rpm
    state: present
  when: ansible_fqdn == 'db04'

- name: Push kibana config                  # 推kibana配置文件
  copy:
    src: kibana.yml
    dest: /etc/kibana/kibana.yml
  when: ansible_fqdn == 'db04'

- name: start kibana                        # 启动kibana
  service:
    name: kibana.service
    state: started
    enabled: yes
  when: ansible_fqdn == 'db04'
  
  
  
  
  
[root@m01 tasks]# cat Yum_Nginx_Tomcat.yml 
- name: Unarchive tomcat packages               # 解压tomcat包到web01的/usr/local下
  unarchive: 
    src: apache-tomcat-10.0.0-M7.tar.gz
    dest: /usr/local/
  when: ansible_fqdn == 'web01'

- name: link tomcat                             # 创建软链接
  file:
    path: /usr/local/tomcat
    src: /usr/local/apache-tomcat-10.0.0-M7
    state: link
  when: ansible_fqdn == 'web01'

- name: start tomcat                    # 启动tomcat (这里一定要写原文件绝对路径并且nohup不加&)
  shell: 'nohup /usr/local/apache-tomcat-10.0.0-M7/bin/startup.sh'
  when: ansible_fqdn == 'web01'

- name: Yum nginx                       # 安装nginx
  yum:
    name: nginx
    state: present
  when: ansible_fqdn == 'web01'

- name: Start nginx                     # 启动nginx
  service:
    name: nginx
    state: started
    enabled: yes
  when: ansible_fqdn == 'web01'
  
  
  
  
  
  
  [root@m01 tasks]# cat Yum_logstash.yml 
- name: Push logstash Packages          # 推logstash安装包
  copy:
    src: logstash-6.6.0.rpm
    dest: /root/logstash-6.6.0.rpm
  when: ansible_fqdn == 'web01'

- name: Yum logstash                    # 安装lgostash
  yum:
    name: /root/logstash-6.6.0.rpm
    state: present
  when: ansible_fqdn == 'web01'

- name: Grant directory                 # 递归授权目录
  file:
    path: /usr/share/logstash/
    owner: logstash
    group: logstash
    state: directory
    recurse: yes
  when: ansible_fqdn == 'web01'

- name: Push logstash config                # 推logstash收集日志配置文件
  copy: 
    src: nginx_tomcat.conf
    dest: /etc/logstash/conf.d/nginx_tomcat.conf
  when: ansible_fqdn == 'web01'

- name: Push logstash sh                    # 推logstash 环境变量文件
  copy:
    src: logstash.sh
    dest: /etc/profile.d/logstash.sh
  when: ansible_fqdn == 'web01'

- name: Source                              # 刷新环境变量
  shell: 'source /etc/profile'
  when: ansible_fqdn == 'web01'

- name: Start logstash                      # 后台不终断启动logstash
  shell: 'nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_tomcat.conf &> /dev/null &'
  when: ansible_fqdn == 'web01'
  
  
  
  
  
# 8.一键开启ELK
[root@m01 ansible_elasticsearch]# ansible-playbook site.yml -i hosts 

dipqc6.png

dipb1x.png


极客公园 , 版权所有丨如未注明 , 均为原创丨本网站采用BY-NC-SA协议进行授权
转载请注明原文链接:ansible实现ELK
喜欢 (0)
[17551054905]
分享 (0)

您必须 登录 才能发表评论!