一、prometheus基礎監控

阿新 • • 發佈：2022-03-04

一、系統環境

root@prometheus:~# cat /etc/issue
Ubuntu 20.04.3 LTS \n \l

192.168.88.200 prometheus-server
192.168.88.201 node1
192.168.88.202 node2

192.168.88.101 k8s-master
192.168.88.102 k8s-node1
192.168.88.103 k8s-node2

二、二進位制安裝prometheus

prometheus官網下載地址：https://prometheus.io/download/

# 解壓二進位制包
root@prometheus:~# tar xf /usr/local/src/prometheus-2.30.3.linux-amd64.tar.gz -C /usr/local/
root@prometheus:~# cd /usr/local/
root@prometheus:/usr/local# ln -sv prometheus-2.30.3.linux-amd64/ prometheus
'prometheus' -> 'prometheus-2.30.3.linux-amd64/'
root@prometheus:/usr/local# cd prometheus
root@prometheus:/usr/local/prometheus# ls
LICENSE  NOTICE  console_libraries  consoles  prometheus  prometheus.yml  promtool
root@prometheus:/usr/local/prometheus#


# prometheus.service 檔案
root@prometheus:~# cat > /etc/systemd/system/prometheus.service << EOF
> [Unit]
> Description=Prometheus Server
> Documentation=https://prometheus.io/docs/introduction/overview
> After=network.target
> 
> [Service]
> Restart=on-failure
> WorkingDirectory=/usr/local/prometheus
> ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml
> 
> [Install]
> WantedBy=multi-user.target
> EOF
root@prometheus:~#


# 啟動
root@prometheus:/usr/local# systemctl daemon-reload
root@prometheus:/usr/local# systemctl start prometheus
root@prometheus:/usr/local# netstat -tnpl | grep 9090
tcp6       0      0 :::9090                 :::*                    LISTEN      2787/prometheus     
root@prometheus:/usr/local#


# prometheus配置檔案
root@prometheus:~# cat /usr/local/prometheus/prometheus.yml
global:
  scrape_interval: 15s # 全域性配置，15s收集一次資料
  evaluation_interval: 15s # 全域性配置，15s評估一次告警規則是否觸發

# 告警元件Alertmanager的配置
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093   

# 告警規則檔案，根據evaluation_interval評估間隔”定期評估
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# 配置指標資料採集
scrape_configs:
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    # 目標主機地址和埠
    static_configs:
      - targets: ["localhost:9090"]

訪問prometheus的9090埠，可通過prometheus:9090/metrics來獲取prometheus伺服器自己的指標資料

三、prometheus採集node指標資料

在node1和node2節點上部署node_exporter

# 在node1和node2上執行以下步驟

# 二進位制解包
root@node1:~# cd /usr/local/src/
root@node1:/usr/local/src# tar xf node_exporter-1.2.2.linux-amd64.tar.gz 
root@node1:/usr/local/src# cd node_exporter-1.2.2.linux-amd64/
root@node1:/usr/local/src/node_exporter-1.2.2.linux-amd64# ls
LICENSE  NOTICE  node_exporter
root@node1:/usr/local/src/node_exporter-1.2.2.linux-amd64# mv node_exporter /usr/local/bin/
root@node1:/usr/local/src/node_exporter-1.2.2.linux-amd64#


# 建立service檔案
root@node1:~# cat > /etc/systemd/system/node-exporter.service << EOF
> [Unit]
> Description=Prometheus Node Exporter
> After=network.target
> 
> [Service]
> ExecStart=/usr/local/bin/node_exporter
> 
> [Install]
> WantedBy=multi-user.target
> EOF
root@node1:~#


# 啟動
root@node1:~# systemctl daemon-reload
root@node1:~# systemctl start node-exporter
root@node1:~# netstat -tnlp | grep 9100
tcp6       0      0 :::9100                 :::*                    LISTEN      2648/node_exporter  
root@node1:~#

訪問node節點的9100:/metrics即可檢視node的指標資料

在prometheus伺服器上配置採集node節點資料

root@prometheus:/usr/local/prometheus# cat prometheus.yml
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]
  
  - job_name: "nodes"
    static_configs:
      - targets:
        - 192.168.88.201:9100
        - 192.168.88.202:9100
        labels:
          system: linux
root@prometheus:/usr/local/prometheus#


# 重啟prometheus
root@prometheus:~# systemctl restart prometheus

訪問prometheus web頁面檢視node1和node2已經新增至prometheus

驗證node資料

四、prometheus採集blackbox exporter指標資料

blackbox_exporter可以通過HTTP、HTTPS、DNS、TCP和ICMP對被監控節點進行監控和資料採集

HTTP/HTTPS：URL/API 可用性檢測
TCP：埠監聽檢測
ICMP：主機存活性檢測
DNS：域名解析檢測

1、部署blackbox_exporter

# 解包
root@node1:/usr/local/src# tar xf blackbox_exporter-0.19.0.linux-amd64.tar.gz -C /usr/local/
root@node1:/usr/local/src# cd /usr/local/
root@node1:/usr/local# ls
bin  blackbox_exporter-0.19.0.linux-amd64  etc  games  include  lib  man  sbin  share  src
root@node1:/usr/local# ln -sv blackbox_exporter-0.19.0.linux-amd64/ blackbox_exporter
'blackbox_exporter' -> 'blackbox_exporter-0.19.0.linux-amd64/'
root@node1:/usr/local#


# service啟動檔案
root@node1:/usr/local/blackbox_exporter# cat > /etc/systemd/system/black-exporter.service << EOF
> [Unit]
> Description=Prometheus Blackbox Exporter
> After=network.target
> 
> [Service]
> Type=simple
> User=root
> Group=root
> ExecStart=/usr/local/blackbox_exporter/blackbox_exporter --config.file=/usr/local/blackbox_exporter/blackbox.yml --web.listen-address=:9115
> Restart=on-failure
> 
> [Install]
> WantedBy=multi-user.target
> EOF


# blackbox配置檔案
root@node1:/usr/local/blackbox_exporter# cat blackbox.yml 
modules:
  http_2xx:
    prober: http
  http_post_2xx:
    prober: http
    http:
      method: POST
  tcp_connect:
    prober: tcp
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
      - send: "SSH-2.0-blackbox-ssh-check"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp


# 啟動
root@node1:/usr/local/blackbox_exporter# systemctl daemon-reload
root@node1:/usr/local/blackbox_exporter# systemctl start black-exporter.service 
root@node1:/usr/local/blackbox_exporter# netstat -tnlp | grep 9115
tcp6       0      0 :::9115                 :::*                    LISTEN      3211/blackbox_expor 
root@node1:/usr/local/blackbox_exporter#

驗證web介面

2、blackbox_exporter實現URL監控

prometheus呼叫blackbox_exporter實現對URL/ICMP的監控

URL監控配置

# prometheus.yml 配置
  - job_name: "http_status"
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets: ['http://www.xiaomi.com','http://www.magedu.com']
        labels:
          instance: http_status
          group: web
    relabel_configs:
      - source_labels: [__address__]   # relabel 通過將__address__（當前目標地址）寫入__param_target標籤來建立一個label
        target_label: __param_target  # 監控目標www.xiaomi.com，作為__address__的value
      - source_labels: [__param_target] # 監控目標
        target_label: url  # 將監控目標與url建立一個label
      - target_label: __address__
        replacement: 192.168.88.201:9115


# 重啟prometheus
root@prometheus:/usr/local/prometheus# ./promtool check config prometheus.yml 
Checking prometheus.yml
  SUCCESS: 0 rule files found

root@prometheus:/usr/local/prometheus# systemctl restart prometheus
root@prometheus:/usr/local/prometheus#

prometheus驗證資料

blackbox_exporter介面驗證資料

3、blackbox_exporter實現ICMP監控

# prometheus.yml 配置
  - job_name: "ping_status"
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets: ['223.6.6.6', '8.8.8.8']
        labels:
          instance: 'ping_status'
          group: 'icmp'
    relabel_configs:
      - source_labels: [__address__] 
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: ip
      - target_label: __address__
        replacement: 192.168.88.201:9115


# 重啟prometheus
root@prometheus:/usr/local/prometheus# ./promtool check config prometheus.yml 
Checking prometheus.yml
  SUCCESS: 0 rule files found

root@prometheus:/usr/local/prometheus# systemctl restart prometheus

prometheus驗證資料

backbox_exporter驗證資料

4、blackbox_exporter實現埠監控

# prometheus.yml 配置
  - job_name: "port_status"
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets: ['192.168.88.201:9100', '192.168.88.201:22']
        labels:
          instance: 'port_status'
          group: 'port'
    relabel_configs:
      - source_labels: [__address__] 
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: ip
      - target_label: __address__
        replacement: 192.168.88.201:9115


# 重啟
root@prometheus:/usr/local/prometheus# ./promtool check config prometheus.yml 
Checking prometheus.yml
  SUCCESS: 0 rule files found

root@prometheus:/usr/local/prometheus# systemctl restart prometheus
root@prometheus:/usr/local/prometheus#

prometheus驗證資料

blackbox_exporter驗證資料

五、grafana配置

1、安裝grafana

# 安裝
root@prometheus:~# apt install adduser libfontconfig1 -y
root@prometheus:~# dpkg -i grafana_7.5.11_amd64.deb


# grafana.ini 配置檔案
root@prometheus:~# vim /etc/grafana/grafana.ini
[server]
# Protocol (http, https, h2, socket)
protocol = http

# The ip address to bind to, empty will bind to all interfaces
http_addr = 0.0.0.0

# The http port  to use
http_port = 3000


# 啟動
root@prometheus:~# systemctl restart grafana-server.service 
root@prometheus:~# systemctl enable grafana-server.service 
Synchronizing state of grafana-server.service with SysV service script with /lib/systemd/systemd-sysv-install.
Executing: /lib/systemd/systemd-sysv-install enable grafana-server
Created symlink /etc/systemd/system/multi-user.target.wants/grafana-server.service → /lib/systemd/system/grafana-server.service.
root@prometheus:~#

登入web介面

新增資料來源