Add creation of PrometheusRules

.cruft.json

@@ -7,7 +7,7 @@
       "name": "icap-virusscan",
       "slug": "icap-virusscan",
       "parameter_key": "icap_virusscan",
-      "test_cases": "defaults with_squid_proxy",
+      "test_cases": "defaults with_squid_proxy monitoring",
       "add_lib": "n",
       "add_pp": "n",
       "add_golden": "y",

.github/workflows/test.yaml

@@ -34,6 +34,7 @@ jobs:
         instance:
           - defaults
           - with_squid_proxy
+          - monitoring
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -50,6 +51,7 @@ jobs:
         instance:
           - defaults
           - with_squid_proxy
+          - monitoring
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

Makefile.vars.mk

@@ -50,4 +50,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml tests/with_squid_proxy.yml
+test_instances = tests/defaults.yml tests/with_squid_proxy.yml tests/monitoring.yml

class/defaults.yml

@@ -13,6 +13,38 @@ parameters:
     namespaceAnnotations: {}
     namespaceLabels: {}
 
+    monitoring:
+      enabled: false
+      prometheusruleGroup:
+        - name: monitoring
+          rules:
+            - alert: KubernetesReplicasetReplicasMismatch
+              annotations:
+                runbook_url: "alert.rubook.com"
+                summary: "Kubernetes ReplicaSet replicas mismatch (instance {{ $labels.instance }})"
+                description: >-
+                  "ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch\n
+                  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+              expr: >-
+                kube_replicaset_spec_replicas{namespace=~"${_instance}"} >
+                kube_replicaset_status_ready_replicas{namespace=~"${_instance}"}
+              for: 2m
+              keep_firing_for: 10m
+              labels:
+                severity: warning
+            - alert: ContainerOOMKilled
+              expr: >-
+                (kube_pod_container_status_restarts_total{namespace="${_instance}"} -
+                kube_pod_container_status_restarts_total{namespace="${_instance}"} offset 10m >= 1)
+                and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled",namespace="${_instance}"}[10m]) == 1
+              for: 2m
+              labels:
+                severity: warning
+              annotations:
+                summary: Kubernetes Container oom killer (instance {{ $labels.instance }})
+                description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+              keep_firing_for: 10m
+
     enable_squid: false
     squid_domain:
     httproute:

component/main.jsonnet

@@ -1,8 +1,10 @@
 // main template for icap-virusscan
 local kap = import 'lib/kapitan.libjsonnet';
 local kube = import 'lib/kube.libjsonnet';
-local inv = kap.inventory();
+local prometheus = import 'lib/prometheus.libsonnet';
+local com = import 'lib/commodore.libjsonnet';
 local sanitizedContainerLib = import 'lib/sanitizedContainer.libsonnet';
+local inv = kap.inventory();
 local sanitizedContainer = sanitizedContainerLib.sanitizedContainer;
 
 // The hiera parameters for the component
@@ -18,10 +20,21 @@ local selectorLabels = {
   instance: instance,
 };
 
-local namespace = kube.Namespace(params.namespace) {
+local namespace = (
+  if params.monitoring.enabled && std.member(inv.applications, 'prometheus') then
+    prometheus.RegisterNamespace(kube.Namespace(params.namespace))
+  else if params.monitoring.enabled && inv.parameters.facts.distribution == 'openshift4' then
+    kube.Namespace(params.namespace) {
+      metadata+: {
+        labels+: { 'openshift.io/cluster-monitoring': 'true' },
+      },
+    }
+  else
+    kube.Namespace(params.namespace)
+) + {
   metadata+: {
-    labels+: params.namespaceLabels,
-    annotations+: params.namespaceAnnotations,
+    labels+: com.makeMergeable(params.namespaceLabels),
+    annotations+: com.makeMergeable(params.namespaceAnnotations),
   },
 };
 
@@ -167,11 +180,29 @@ local networkPolicies = {
   },
 };
 
+local prometheusRule = {
+  apiVersion: 'monitoring.coreos.com/v1',
+  kind: 'PrometheusRule',
+  metadata: {
+    name: 'monitoringRules',
+    namespace: params.namespace,
+    labels: selectorLabels,
+  },
+  spec: {
+  	groups: if std.objectHas( params.monitoring, 'syn_team')
+		then
+			[params.monitoring.prometheusruleGroup[0] + {labels: {syn_team: params.monitoring.syn_team}}]
+		else
+			params.monitoring.prometheusruleGroup
+  }
+};
+
 {
   [if params.createNamespace then '00_namespace']: namespace,
   '01_deployment': deployment,
   [if params.replicas > 1 then '02_podDisruptionBudget']: podDisruptionBudget,
   '03_service': service,
   [if hasNetworkPolicies then '04_networkPolicies']: networkPolicies,
+  [if params.monitoring.enabled then '05_prometheusRule']: prometheusRule,
 } + (import 'lib/testSetup.libsonnet')
 + (import 'lib/debug.libsonnet')

tests/golden/monitoring/monitoring/apps/monitoring.yaml

@@ -0,0 +1,4 @@
+spec:
+  syncPolicy:
+    syncOptions:
+      - ServerSideApply=true

tests/golden/monitoring/monitoring/monitoring/00_namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations: {}
+  labels:
+    name: monitoring
+    openshift.io/cluster-monitoring: 'true'
+  name: monitoring

tests/golden/monitoring/monitoring/monitoring/01_deployment.yaml

@@ -0,0 +1,107 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: clamav-icap
+    instance: monitoring
+  name: clamav-icap
+  namespace: monitoring
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: clamav-icap
+      instance: monitoring
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        app: clamav-icap
+        instance: monitoring
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: clamav-icap
+                    instance: monitoring
+                topologyKey: kubernetes.io/hostname
+              weight: 1
+      containers:
+        - env: []
+          image: ghcr.io/vshn/clamav:20260420_095651
+          livenessProbe:
+            failureThreshold: 3
+            periodSeconds: 5
+            tcpSocket:
+              port: 3310
+            timeoutSeconds: 5
+          name: clamav
+          ports:
+            - containerPort: 3310
+              name: clamav
+          readinessProbe:
+            failureThreshold: 1
+            initialDelaySeconds: 0
+            periodSeconds: 5
+            successThreshold: 1
+            tcpSocket:
+              port: 3310
+            timeoutSeconds: 3
+          resources:
+            limits:
+              cpu: 2000m
+              memory: 3072Mi
+            requests:
+              cpu: 120m
+              memory: 1280Mi
+          startupProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 0
+            periodSeconds: 5
+            tcpSocket:
+              port: 3310
+            timeoutSeconds: 5
+        - env:
+            - name: CLAMD_IP
+              value: 127.0.0.1
+            - name: CLAMD_MAXSIZE
+              value: 50m
+          image: ghcr.io/vshn/c-icap:20260420_095301
+          livenessProbe:
+            failureThreshold: 3
+            periodSeconds: 5
+            tcpSocket:
+              port: 1344
+            timeoutSeconds: 5
+          name: c-icap
+          ports:
+            - containerPort: 1344
+              name: icap
+          readinessProbe:
+            failureThreshold: 1
+            initialDelaySeconds: 0
+            periodSeconds: 5
+            successThreshold: 1
+            tcpSocket:
+              port: 1344
+            timeoutSeconds: 3
+          resources:
+            limits:
+              cpu: 500m
+              memory: 500Mi
+            requests:
+              cpu: 120m
+              memory: 200Mi
+          startupProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 0
+            periodSeconds: 5
+            tcpSocket:
+              port: 1344
+            timeoutSeconds: 5

tests/golden/monitoring/monitoring/monitoring/02_podDisruptionBudget.yaml

@@ -0,0 +1,15 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  labels:
+    app: clamav-icap
+    instance: monitoring
+  name: clamav-icap
+  namespace: monitoring
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: clamav-icap
+      instance: monitoring
+  unhealthyPodEvictionPolicy: IfHealthyBudget

tests/golden/monitoring/monitoring/monitoring/03_service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: clamav-icap
+    instance: monitoring
+  name: icap
+  namespace: monitoring
+spec:
+  ports:
+    - name: icap
+      port: 80
+      protocol: TCP
+      targetPort: icap
+  selector:
+    app: clamav-icap
+    instance: monitoring
+  type: ClusterIP

tests/golden/monitoring/monitoring/monitoring/05_prometheusRule.yaml

@@ -0,0 +1,42 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app: clamav-icap
+    instance: monitoring
+  name: monitoringRules
+  namespace: monitoring
+spec:
+  groups:
+    - labels:
+        syn_team: team_monitoring
+      name: monitoring
+      rules:
+        - alert: KubernetesReplicasetReplicasMismatch
+          annotations:
+            description: '"ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset
+              }} replicas mismatch\n VALUE = {{ $value }}\n  LABELS = {{ $labels }}"'
+            runbook_url: alert.rubook.com
+            summary: Kubernetes ReplicaSet replicas mismatch (instance {{ $labels.instance
+              }})
+          expr: kube_replicaset_spec_replicas{namespace=~"monitoring"} > kube_replicaset_status_ready_replicas{namespace=~"monitoring"}
+          for: 2m
+          keep_firing_for: 10m
+          labels:
+            severity: warning
+        - alert: ContainerOOMKilled
+          annotations:
+            description: |-
+              Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.
+                VALUE = {{ $value }}
+                LABELS = {{ $labels }}
+            summary: Kubernetes Container oom killer (instance {{ $labels.instance
+              }})
+          expr: (kube_pod_container_status_restarts_total{namespace="monitoring"}
+            - kube_pod_container_status_restarts_total{namespace="monitoring"} offset
+            10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled",namespace="monitoring"}[10m])
+            == 1
+          for: 2m
+          keep_firing_for: 10m
+          labels:
+            severity: warning

tests/golden/with_squid_proxy/with_squid_proxy/with_squid_proxy/99_debug.yaml

@@ -189,6 +189,30 @@ data:
       gatewayNamespace: "gateway-dev-ns"
       sectionName: "http-default"
     minAvailable: 1
+    monitoring:
+      enabled: false
+      prometheusruleGroup:
+      - name: "monitoring"
+        rules:
+        - alert: "KubernetesReplicasetReplicasMismatch"
+          annotations:
+            description: "\"ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch\\n VALUE = {{ $value }}\\n  LABELS = {{ $labels }}\""
+            runbook_url: "alert.rubook.com"
+            summary: "Kubernetes ReplicaSet replicas mismatch (instance {{ $labels.instance }})"
+          expr: "kube_replicaset_spec_replicas{namespace=~\"with_squid_proxy\"} > kube_replicaset_status_ready_replicas{namespace=~\"with_squid_proxy\"}"
+          for: "2m"
+          keep_firing_for: "10m"
+          labels:
+            severity: "warning"
+        - alert: "ContainerOOMKilled"
+          annotations:
+            description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+            summary: "Kubernetes Container oom killer (instance {{ $labels.instance }})"
+          expr: "(kube_pod_container_status_restarts_total{namespace=\"with_squid_proxy\"} - kube_pod_container_status_restarts_total{namespace=\"with_squid_proxy\"} offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\",namespace=\"with_squid_proxy\"}[10m]) == 1"
+          for: "2m"
+          keep_firing_for: "10m"
+          labels:
+            severity: "warning"
     namespace: "with_squid_proxy"
     namespaceAnnotations: {}
     namespaceLabels: {}

tests/monitoring.yml

@@ -0,0 +1,9 @@
+---
+parameters:
+  facts:
+    distribution: openshift4
+
+  icap_virusscan:
+    monitoring:
+      enabled: true
+      syn_team: "team_monitoring"