Monitoring the cluster using prometheus.

Prometheus is an open source framework and hence can be used to monitor running kubernetes clusters.

Below steps need to be followed to enable prometheus monitoring for kubernetes.

Step1: Create a yaml file say clusterrole.yaml and paste below contents there-

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRole

metadata:

  name: prometheus

rules:

- apiGroups: [""]

  resources:

  - nodes

  - nodes/proxy

  - services

  - endpoints

  - pods

  verbs: ["get", "list", "watch"]

- apiGroups:

  - extensions

  resources:

  - ingresses

  verbs: ["get", "list", "watch"]

- nonResourceURLs: ["/metrics"]

  verbs: ["get"]

---

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRoleBinding

metadata:

  name: prometheus

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: ClusterRole

  name: prometheus

subjects:

- kind: ServiceAccount

  name: default

  namespace: monitoring

Step2: Creation of role using below command.

kubectl create -f clusterrole.yaml

Step 3: Create confimap.yaml and copy below contents

apiVersion: v1

kind: ConfigMap

metadata:

  name: prometheus-server-conf

  labels:

    name: prometheus-server-conf

  namespace: monitoring

data:

  prometheus.rules: |-

    groups:

    - name: devopscube demo alert

      rules:

      - alert: High Pod Memory

        expr: sum(container_memory_usage_bytes) > 1

        for: 1m

        labels:

          severity: slack

        annotations:

          summary: High Memory Usage

  prometheus.yml: |-

    global:

      scrape_interval: 5s

      evaluation_interval: 5s

    rule_files:

      - /etc/prometheus/prometheus.rules

    alerting:

      alertmanagers:

      - scheme: http

        static_configs:

        - targets:

          - "alertmanager.monitoring.svc:9093"

    scrape_configs:

      - job_name: 'kubernetes-apiservers'

        kubernetes_sd_configs:

        - role: endpoints

        scheme: https

        tls_config:

          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        relabel_configs:

        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]

          action: keep

          regex: default;kubernetes;https

      - job_name: 'kubernetes-nodes'

        scheme: https

        tls_config:

          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        kubernetes_sd_configs:

        - role: node

        relabel_configs:

        - action: labelmap

          regex: __meta_kubernetes_node_label_(.+)

        - target_label: __address__

          replacement: kubernetes.default.svc:443

        - source_labels: [__meta_kubernetes_node_name]

          regex: (.+)

          target_label: __metrics_path__

          replacement: /api/v1/nodes/${1}/proxy/metrics

      - job_name: 'kubernetes-pods'

        kubernetes_sd_configs:

        - role: pod

        relabel_configs:

        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]

          action: keep

          regex: true

        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]

          action: replace

          target_label: __metrics_path__

          regex: (.+)

        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]

          action: replace

          regex: ([^:]+)(?::d+)?;(d+)

          replacement: $1:$2

          target_label: __address__

        - action: labelmap

          regex: __meta_kubernetes_pod_label_(.+)

        - source_labels: [__meta_kubernetes_namespace]

          action: replace

          target_label: kubernetes_namespace

        - source_labels: [__meta_kubernetes_pod_name]

          action: replace

          target_label: kubernetes_pod_name

      - job_name: 'kube-state-metrics'

        static_configs:

          - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080']

      - job_name: 'kubernetes-cadvisor'

        scheme: https

        tls_config:

          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        kubernetes_sd_configs:

        - role: node

        relabel_configs:

        - action: labelmap

          regex: __meta_kubernetes_node_label_(.+)

        - target_label: __address__

          replacement: kubernetes.default.svc:443

        - source_labels: [__meta_kubernetes_node_name]

          regex: (.+)

          target_label: __metrics_path__

          replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

      - job_name: 'kubernetes-service-endpoints'

        kubernetes_sd_configs:

        - role: endpoints

        relabel_configs:

        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]

          action: keep

          regex: true

        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]

          action: replace

          target_label: __scheme__

          regex: (https?)

        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]

          action: replace

          target_label: __metrics_path__

          regex: (.+)

        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]

          action: replace

          target_label: __address__

          regex: ([^:]+)(?::d+)?;(d+)

          replacement: $1:$2

        - action: labelmap

          regex: __meta_kubernetes_service_label_(.+)

        - source_labels: [__meta_kubernetes_namespace]

          action: replace

          target_label: kubernetes_namespace

        - source_labels: [__meta_kubernetes_service_name]

          action: replace

          target_label: kubernetes_name

Step4: Create configmap

# kubectl create -f configmap.yaml

Step5: Create prometheus deployment

Create file prometheus_deployment.yaml and copy below lines

apiVersion: extensions/v1beta1

kind: Deployment

metadata:

  name: prometheus-deployment

  namespace: monitoring

spec:

  replicas: 1

  template:

    metadata:

      labels:

        app: prometheus-server

    spec:

      containers:

        - name: prometheus

          image: prom/prometheus:v2.12.0

          args:

            - "--config.file=/etc/prometheus/prometheus.yml"

            - "--storage.tsdb.path=/prometheus/"

          ports:

            - containerPort: 9090

          volumeMounts:

            - name: prometheus-config-volume

              mountPath: /etc/prometheus/

            - name: prometheus-storage-volume

              mountPath: /prometheus/

      volumes:

        - name: prometheus-config-volume

          configMap:

            defaultMode: 420

            name: prometheus-server-conf

        - name: prometheus-storage-volume

          emptyDir: {}

Step6: Create

# kubectl create -f prometheus_deployment.yaml

Step7: Create deployment using below command

# kubectl get deployments --namespace=monitoring

Step8: Exposing prometheus as a service

Creation of prometheus_service.yaml and edit below lines -

 apiVersion: v1

kind: Service

metadata:

  name: prometheus-service

  namespace: monitoring

  annotations:

      prometheus.io/scrape: 'true'

      prometheus.io/port: '9090'

spec:

  selector:

    app: prometheus-server

  type: NodePort

  ports:

    - port: 8080

      targetPort: 9090

      nodePort: 30000

Use below command to create service

kubectl create -f prometheus-service.yaml --namespace=monitoring

Access the dashboard -