Cluster Alerts

Kubernetes prom-node-exporter daemonset:

---
###
## prom-node-exporter
###
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  namespace: monitoring-ns
  name: prom-node-exporter-ds
  labels:
    tier: monitoring
    app: prom-node-exporter
spec:
  selector:
    matchLabels:
      tier: monitoring
      app: prom-node-exporter
  template:
    metadata:
      labels:
        tier: monitoring
        app: prom-node-exporter
      annotations:
        prometheus.io/port: "9100"
        prometheus.io/scrape: "true"
    spec:
      containers:
        - name: prometheus-node-exporter
          image: prom/node-exporter:latest
          ports:
            - containerPort: 9100
          securityContext:
            privileged: true
          args:
            - '--path.procfs=/host/proc'
            - '--path.sysfs=/host/sys'
            - '--path.rootfs=/host/rootfs'
          volumeMounts:
            - mountPath: /host/proc
              name: proc-fs
            - mountPath: /host/sys
              name: sys-fs
            - mountPath: /host/rootfs
              name: root-fs
      hostNetwork: true
      hostPID: true
      volumes:
        - name: proc-fs
          hostPath:
            path: /proc
        - name: sys-fs
          hostPath:
            path: /sys
        - name: root-fs
          hostPath:
            path: /
---
apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: monitoring-ns
  name: prom-node-exporter-sa
---

Prometheus.yaml

global:
  scrape_interval: 10s
  scrape_timeout: 10s
  evaluation_interval: 30s
rule_files:
- /etc/config/rules/*.yaml

scrape_configs:


  - job_name: prometheus-node_exporter
    honor_labels: true
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        separator: ;
        regex: "true"
        replacement: $1
        action: keep
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        separator: ;
        regex: (.+)
        target_label: __metrics_path__
        replacement: $1
        action: replace
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_name]
        separator: ;
        regex: ^(.+;.*)|(;.*metrics)$
        replacement: $1
        action: keep
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        separator: ;
        regex: (.+):(?:\d+);(\d+)
        target_label: __address__
        replacement: ${1}:${2}
        action: replace
      - separator: ;
        regex: __meta_kubernetes_pod_label_(.+)
        replacement: $1
        action: labelmap
      - source_labels: [__meta_kubernetes_pod_node_name]
        separator: ;
        regex: (.*)
        target_label: node_name
        replacement: $1
        action: replace