实战Prometheusblackboxexporter黑盒监控

　　一、Blackbox-exporter 安装部署1.1、Blackbox配置文件[root@mayi-ops ~]# cat blackbox-export-cpnfogmap.yaml  apiVersion: v1 kind: ConfigMap metadata:   name: blackbox-exporter   namespace: kube-system   labels:     app: blackbox-exporter data:   blackbox.yml: |-     modules:       ## ----------- DNS 检测配置 -----------       dns_tcp:           prober: dns         dns:           transport_protocol: ＂tcp＂           preferred_ip_protocol: ＂ip4＂           query_name: ＂kubernetes.default.svc.cluster.local＂ # 用于检测域名可用的网址           query_type: ＂A＂        ## ----------- TCP 检测模块配置 -----------       tcp_connect:         prober: tcp         timeout: 5s       ## ----------- ICMP 检测配置 -----------       ping:         prober: icmp         timeout: 5s         icmp:           preferred_ip_protocol: ＂ip4＂       ## ----------- HTTP GET 2xx 检测模块配置 -----------       http_get_2xx:           prober: http         timeout: 10s         http:           method: GET           preferred_ip_protocol: ＂ip4＂           valid_http_versions: [＂HTTP/1.1＂,＂HTTP/2＂]           valid_status_codes: [200]           # 验证的HTTP状态码,默认为2xx           no_follow_redirects: false          # 是否不跟随重定向       ## ----------- HTTP GET 3xx 检测模块配置 -----------       http_get_3xx:           prober: http         timeout: 10s         http:           method: GET           preferred_ip_protocol: ＂ip4＂           valid_http_versions: [＂HTTP/1.1＂,＂HTTP/2＂]           valid_status_codes: [301,302,304,305,306,307]  # 验证的HTTP状态码,默认为2xx           no_follow_redirects: false                     # 是否不跟随重定向       ## ----------- HTTP POST 监测模块 -----------       http_post_2xx:          prober: http         timeout: 10s         http:           method: POST           preferred_ip_protocol: ＂ip4＂           valid_http_versions: [＂HTTP/1.1＂, ＂HTTP/2＂]           #headers:                             # HTTP头设置           #  Content-Type: application/json           #body: ＂{}＂                           # 请求体设置 1.2、blackbox-exporter部署yaml[root@mayi-ops ~]# cat blackbox-exporter-deployment.yaml apiVersion: v1 kind: Service metadata:   name: blackbox-exporter   namespace: kube-system   labels:     k8s-app: blackbox-exporter spec:   type: ClusterIP   ports:   - name: http     port: 9115     targetPort: 9115   selector:     k8s-app: blackbox-exporter --- apiVersion: apps/v1 kind: Deployment metadata:   name: blackbox-exporter   namespace: kube-system   labels:     k8s-app: blackbox-exporter spec:   replicas: 1   selector:     matchLabels:       k8s-app: blackbox-exporter   template:     metadata:       labels:         k8s-app: blackbox-exporter     spec:       containers:       - name: blackbox-exporter         image: prom/blackbox-exporter:v0.19.0         args:         - --config.file=/etc/blackbox_exporter/blackbox.yml         - --web.listen-address=:9115         - --log.level=info         ports:         - name: http           containerPort: 9115         resources:           limits:             cpu: 200m             memory: 256Mi           requests:             cpu: 100m             memory: 50Mi         livenessProbe:           tcpSocket:             port: 9115           initialDelaySeconds: 5           timeoutSeconds: 5           periodSeconds: 10           successThreshold: 1           failureThreshold: 3         readinessProbe:           tcpSocket:             port: 9115           initialDelaySeconds: 5           timeoutSeconds: 5           periodSeconds: 10           successThreshold: 1           failureThreshold: 3         volumeMounts:         - name: config           mountPath: /etc/blackbox_exporter       volumes:       - name: config         configMap:           name: blackbox-exporter           defaultMode: 420 二、prometheus 添加scrape_configs配置    #监控svc     - job_name: kubernetes-services       metrics_path: /probe       params:         module:         - ＂http_get_2xx＂         - ＂http_get_3xx＂       kubernetes_sd_configs:       - role: service       relabel_configs:       - action: keep         source_labels: [__meta_kubernetes_service_annotation_prometheus_io_http_probe]         regex: ＂true＂       - action: replace         source_labels:          - ＂__meta_kubernetes_service_name＂         - ＂__meta_kubernetes_namespace＂         - ＂__meta_kubernetes_service_annotation_prometheus_io_http_probe_port＂         - ＂__meta_kubernetes_service_annotation_prometheus_io_http_probe_path＂         target_label: __param_target         regex: (.+);(.+);(.+);(.+)         replacement: $1.$2:$3$4       - target_label: __address__         replacement: blackbox-exporter.kube-system.svc.cluster.local:9115       - source_labels: [__param_target]         target_label: instance       - action: labelmap         regex: __meta_kubernetes_service_label_(.+)       - source_labels: [__meta_kubernetes_namespace]         target_label: kubernetes_namespace       - source_labels: [__meta_kubernetes_service_name]         target_label: kubernetes_name  三、svc http监控apiVersion: v1 kind: Service metadata:   name: nginx   namespace: mayi   annotations:                  #添加label，提供给prometheus访问     prometheus.io/http-probe: ＂true＂                prometheus.io/http-probe-path: /     prometheus.io/http-probe-port: ＂80＂ spec:   type: NodePort   selector:     app: nginx   ports:   - port: 80     protocol: TCP     targetPort: 80     nodePort: 31888 四、core-dns监控    #在prometheus 添加scrape_configs配置     #监控kube-dns     - job_name: ＂kubernetes-dns＂       metrics_path: /probe       params:         module: [dns_tcp]       static_configs:         - targets:           - kube-dns.kube-system:53      #kube-dns访问地址       relabel_configs:         - source_labels: [__address__]           target_label: __param_target         - source_labels: [__param_target]           target_label: instance         - target_label: __address__           replacement: blackbox-exporter.kube-system.svc.cluster.local:9115   #将kube-dns访问地址，提供给blackbox-exporter，并使用dns_tcp模块检查 五、TCP 端口监控    #在prometheus 添加scrape_configs配置     #监控K8S集群外服务端口     - job_name: ＂port_test＂       scrape_interval: 30s       metrics_path: /probe       params:         module: [tcp_connect]       static_configs:       - targets: [＂192.168.0.99:8082＂]         labels:           group: ＂superset＂   #添加label,便于告警区分       - targets: [＂192.168.0.99:8080＂]         labels:           group: ＂datax＂       - targets: [＂192.168.0.99:8888＂]         labels:           group: ＂Jupyter＂       relabel_configs:         - source_labels: [__address__]           target_label: __param_target         - source_labels: [__param_target]           target_label: instance         - target_label: __address__           replacement: blackbox-exporter.kube-system.svc.cluster.local:9115 #将以上地址，提供给blackbox-exporter，并使用tcp_connect模块检查 六、prometheus 告警规则#以下为Blackbox-exporer通用告警模板，可根据label自行调整 blackbox.rules: |     groups:     - name: blackbox.rules       rules:       - alert: blackbox_network_stats         expr: probe_success == 0         for: 1m         labels:           team: ops           severity: critical         annotations:           description: ＂接口/主机/路径/端口：{{ $labels.instance }} 无法连通＂