forust/homelab
0
0 Fork 0
Code Issues 1 Pull Requests 3 Actions Packages Wiki Activity

feat(observability): cert-manager metrics, dashboard and alerts
ci / lint-prettier (push) Successful in 3s
Details
ci / lint-ruff (push) Successful in 1s
Details
ci / lint-yaml (push) Successful in 2s
Details
ci / lint-dockerfiles (push) Successful in 1s
Details
ci / validate (push) Successful in 2s
Details
renovate-ci / validate-renovate (push) Successful in 13s
Details
ci / build (push) Successful in 1s
Details
ci / deploy-userbot-panel (push) Skipped
Details

Browse Source
This commit is contained in:
forust committed 2026-09-23 14:32:36 +02:00
2 parents 6be3769288 1cbdfc1d6f
commit 1f7166026b
3 files changed
+965
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cert-manager/k8s/cert-manager-values.yaml
+7
View File
@@ -1,2 +1,9 @@
crds:
enabled: true
prometheus:
servicemonitor:
enabled: true
interval: 60s
scrapeTimeout: 30s
labels:
release: prometheus-stack
prometheus-stack/k8s/cert-manager-alerts.yaml
+43
View File
@@ -0,0 +1,43 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: cert-manager
namespace: prometheus
labels:
release: prometheus-stack
spec:
groups:
- name: cert-manager
rules:
- alert: CertManagerCertNotReady
expr: certmanager_certificate_ready_status{condition="False"} == 1
for: 10m
labels:
severity: critical
annotations:
summary: "Certificate {{ $labels.exported_namespace }}/{{ $labels.name }} is not ready"
description: "Certificate has been failing for more than 10 minutes. Check Challenges/Orders in that namespace."
- alert: CertManagerCertExpirySoon
expr: certmanager_certificate_expiration_timestamp_seconds - time() < 86400 * 14
for: 1h
labels:
severity: warning
annotations:
summary: "Certificate {{ $labels.exported_namespace }}/{{ $labels.name }} expires in less than 14 days"
description: "Renewal should have happened automatically. Check cert-manager logs if it persists."
- alert: CertManagerCertExpiryCritical
expr: certmanager_certificate_expiration_timestamp_seconds - time() < 86400 * 7
for: 1h
labels:
severity: critical
annotations:
summary: "Certificate {{ $labels.exported_namespace }}/{{ $labels.name }} expires in less than 7 days"
description: "Manual intervention likely needed: cmctl status certificate {{ $labels.name }} -n {{ $labels.exported_namespace }}."
- alert: CertManagerHittingRateLimits
expr: sum by (exported_namespace) (rate(certmanager_http_acme_client_request_count{status=~"429.*"}[10m])) > 0
for: 15m
labels:
severity: warning
annotations:
summary: "cert-manager is hitting ACME rate limits"
description: "Back off manual issuance retries and check failed Orders/Challenges."
prometheus-stack/k8s/dashboards/cert-manager-dashboard.yaml
+915
View File
@@ -0,0 +1,915 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-dashboard-cert-manager
namespace: prometheus
labels:
grafana_dashboard: "1"
app: cert-manager
data:
cert-manager.json: |-
{
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "11.2.0"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "stat",
"name": "Stat",
"version": ""
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "The dashboard gives an overview of the SSL certs managed by cert-manager in Kubernetes",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 22908,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"default": false,
"type": "prometheus",
"uid": "${datasource}"
},
"description": "The number if available certificates",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 3,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "value",
"wideLayout": true
},
"pluginVersion": "11.2.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "count(certmanager_certificate_ready_status{condition=\"True\", namespace=~\"$namespace\"})",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A",
"useBackend": false
}
],
"title": "Valid Certificates",
"type": "stat"
},
{
"datasource": {
"name": "${datasource}",
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 9,
"x": 3,
"y": 0
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.2.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(certmanager_certificate_ready_status{namespace=~\"$namespace\"}) by (condition,name)>0",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{name}}:{{condition}}",
"range": true,
"refId": "A",
"step": 1800
}
],
"title": "Certificate Ready Status",
"type": "timeseries"
},
{
"datasource": {
"default": false,
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.2.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(certmanager_http_acme_client_request_count{namespace=~\"$namespace\"}[2m])) BY (status, path)>0",
"legendFormat": "{{status}}:{{path}}",
"range": true,
"refId": "A"
}
],
"title": "Overall requests per 2m",
"type": "timeseries"
},
{
"datasource": {
"default": false,
"type": "prometheus",
"uid": "${datasource}"
},
"description": "The number of certificates that will expire within the next 14 days",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "#EAB839",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 3,
"x": 0,
"y": 3
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "count(certmanager_certificate_expiration_timestamp_seconds{namespace=~\"$namespace\"} < (time()+(14*24*3600)))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "{{exported_namespace}}/{{name}}",
"range": false,
"refId": "A",
"useBackend": false
}
],
"title": "Expiring Certificates",
"type": "stat"
},
{
"datasource": {
"default": false,
"type": "prometheus",
"uid": "${datasource}"
},
"description": "Total number of HTTP requests",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "text",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 3,
"w": 3,
"x": 0,
"y": 6
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.2.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "sum(certmanager_http_acme_client_request_count)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A",
"useBackend": false
}
],
"title": "Total ACME Requests",
"type": "stat"
},
{
"datasource": {
"default": false,
"type": "prometheus",
"uid": "${datasource}"
},
"description": "Time before the certificates expire",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 14
},
{
"color": "green",
"value": 30
},
{
"color": "dark-green",
"value": 60
}
]
},
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "证书有效期（天）"
},
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "semi-dark-red",
"value": 10
},
{
"color": "yellow",
"value": 15
},
{
"color": "green",
"value": 25
}
]
}
},
{
"id": "custom.align",
"value": "center"
},
{
"id": "custom.cellOptions",
"value": {
"mode": "gradient",
"type": "color-background"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "自动续费证书时间（天）"
},
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 10
},
{
"color": "yellow",
"value": 15
},
{
"color": "green",
"value": 20
}
]
}
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-background"
}
}
]
}
]
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 9
},
"id": 5,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "11.2.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": false,
"expr": "sort_desc(certmanager_certificate_expiration_timestamp_seconds{namespace=~\"$namespace\"} - time())/(24*3600)",
"format": "table",
"instant": true,
"legendFormat": "{{name}}",
"range": false,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": false,
"expr": "sort(certmanager_certificate_renewal_timestamp_seconds{namespace=~\"$namespace\"} - time())/(24*3600)",
"format": "table",
"hide": false,
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "B"
}
],
"title": "证书有效期",
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "name",
"mode": "outer"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time 1": false,
"instance 1": false,
"issuer_group 1": false,
"issuer_name 1": false,
"job 1": false,
"namespace 1": false
},
"includeByName": {},
"indexByName": {
"Time 1": 4,
"Time 2": 10,
"Value #A": 2,
"Value #B": 3,
"instance 1": 5,
"instance 2": 11,
"issuer_group 1": 8,
"issuer_group 2": 16,
"issuer_kind 1": 9,
"issuer_kind 2": 12,
"issuer_name 1": 6,
"issuer_name 2": 13,
"job 1": 7,
"job 2": 14,
"name": 0,
"namespace 1": 1,
"namespace 2": 15
},
"renameByName": {
"Value #A": "证书有效期（天）",
"Value #B": "自动续费证书时间（天）"
}
}
},
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": [
"name",
"namespace 1",
"Value #A",
"Value #B"
]
}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "Time before the certificates expire",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"api-elnkpro-ssl",
"alidns-webhook-tls"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 9
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "(certmanager_certificate_expiration_timestamp_seconds{namespace=~\"$namespace\"} - time())/(24*3600)",
"instant": false,
"legendFormat": "{{name}}",
"range": true,
"refId": "A"
}
],
"title": "证书到期时间",
"type": "timeseries"
}
],
"schemaVersion": 39,
"tags": [
"k8s",
"cert-manager"
],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "/^.*k8s/",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(certmanager_certificate_expiration_timestamp_seconds,namespace)",
"hide": 0,
"includeAll": true,
"multi": false,
"name": "namespace",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(certmanager_certificate_expiration_timestamp_seconds,namespace)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Cert-manager-Kubernetes",
"uid": "cert-manager-overview",
"version": 15,
"weekStart": ""
}