Add Alertmanager role for Prometheus alerting

- Docker Compose deployment for Alertmanager v0.27.0
- Optional Discord webhook integration for notifications
- Persistent storage for alert state

roles/alertmanager/tasks/main.yml

@@ -0,0 +1,84 @@
+---
+
+- name: Read Alertmanager Slack webhook URL
+  set_fact:
+    alertmanager_slack_webhook_url: "{{ ALERTMANAGER_SLACK_WEBHOOK_URL | default(lookup('env', 'ALERTMANAGER_SLACK_WEBHOOK_URL') | default('', true), true) }}"
+  no_log: true
+
+- name: Read Alertmanager Discord webhook URL
+  set_fact:
+    alertmanager_discord_webhook_url: "{{ ALERTMANAGER_DISCORD_WEBHOOK_URL | default(lookup('env', 'ALERTMANAGER_DISCORD_WEBHOOK_URL') | default('', true), true) }}"
+  no_log: true
+
+- name: Fail if Slack webhook URL is configured in Discord webhook variable
+  assert:
+    that:
+      - not (alertmanager_discord_webhook_url is match('^https://hooks\\.slack\\.com/'))
+    fail_msg: >-
+      ALERTMANAGER_DISCORD_WEBHOOK_URL appears to be a Slack webhook (hooks.slack.com).
+      Move it to ALERTMANAGER_SLACK_WEBHOOK_URL and clear ALERTMANAGER_DISCORD_WEBHOOK_URL.
+  when: alertmanager_discord_webhook_url | length > 0
+
+- name: Fail if Discord webhook URL is configured in Slack webhook variable
+  assert:
+    that:
+      - not (alertmanager_slack_webhook_url is match('^https://((ptb|canary)\\.)?discord(app)?\\.com/api/webhooks/'))
+    fail_msg: >-
+      ALERTMANAGER_SLACK_WEBHOOK_URL appears to be a Discord webhook.
+      Move it to ALERTMANAGER_DISCORD_WEBHOOK_URL and clear ALERTMANAGER_SLACK_WEBHOOK_URL.
+  when: alertmanager_slack_webhook_url | length > 0
+
+- name: Fail if no Alertmanager webhook is configured
+  assert:
+    that:
+      - (alertmanager_slack_webhook_url | length > 0) or (alertmanager_discord_webhook_url | length > 0)
+    fail_msg: "Set ALERTMANAGER_SLACK_WEBHOOK_URL or ALERTMANAGER_DISCORD_WEBHOOK_URL"
+
+- name: Fail if both Slack and Discord webhooks are configured
+  assert:
+    that:
+      - not ((alertmanager_slack_webhook_url | length > 0) and (alertmanager_discord_webhook_url | length > 0))
+    fail_msg: "Configure only one of ALERTMANAGER_SLACK_WEBHOOK_URL or ALERTMANAGER_DISCORD_WEBHOOK_URL"
+
+- name: Determine Alertmanager receiver type
+  set_fact:
+    alertmanager_receiver_type: "{{ 'slack' if (alertmanager_slack_webhook_url | length > 0) else 'discord' }}"
+
+- name: Fail if selected receiver has an invalid webhook URL
+  assert:
+    that:
+      - (alertmanager_receiver_type != 'slack') or (alertmanager_slack_webhook_url is match('^https://hooks\\.slack\\.com/'))
+      - (alertmanager_receiver_type != 'discord') or (alertmanager_discord_webhook_url is match('^https://((ptb|canary)\\.)?discord(app)?\\.com/api/webhooks/'))
+    fail_msg: >-
+      Alertmanager webhook URL does not match expected format for receiver type '{{ alertmanager_receiver_type }}'.
+      Slack expects https://hooks.slack.com/... and Discord expects https://discord.com/api/webhooks/....
+
+- name: Create Alertmanager directory
+  file:
+    path: /opt/alertmanager
+    state: directory
+
+- name: Ensure monitoring network exists
+  command: docker network inspect monitoring
+  register: monitoring_network
+  changed_when: false
+  failed_when: false
+
+- name: Create monitoring network if missing
+  command: docker network create monitoring
+  when: monitoring_network.rc != 0
+
+- name: Copy Alertmanager configuration
+  template:
+    src: alertmanager.yml.j2
+    dest: /opt/alertmanager/alertmanager.yml
+
+- name: Copy Docker Compose file for Alertmanager
+  template:
+    src: docker-compose.yml.j2
+    dest: /opt/alertmanager/docker-compose.yml
+
+- name: Deploy Alertmanager
+  command: docker compose up -d
+  args:
+    chdir: /opt/alertmanager

roles/alertmanager/templates/alertmanager.yml.j2

@@ -0,0 +1,23 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  group_by: ['alertname']
+  group_wait: 20s
+  group_interval: 5m
+  repeat_interval: 3h
+  receiver: primary
+
+receivers:
+  - name: primary
+{% if alertmanager_receiver_type == 'slack' %}
+    slack_configs:
+      - api_url: "{{ alertmanager_slack_webhook_url }}"
+        send_resolved: true
+        channel: "{{ ALERTMANAGER_SLACK_CHANNEL | default(lookup('env', 'ALERTMANAGER_SLACK_CHANNEL') | default('#alerts', true), true) }}"
+        username: "{{ ALERTMANAGER_SLACK_USERNAME | default(lookup('env', 'ALERTMANAGER_SLACK_USERNAME') | default('alertmanager', true), true) }}"
+{% else %}
+    webhook_configs:
+      - url: http://alertmanager-discord:9094
+        send_resolved: true
+{% endif %}

roles/alertmanager/templates/docker-compose.yml.j2

@@ -0,0 +1,36 @@
+services:
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+      - --storage.path=/alertmanager
+    volumes:
+      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager_data:/alertmanager
+    networks:
+      - monitoring
+    restart: unless-stopped
+    labels:
+      - com.centurylinklabs.watchtower.enable=true
+
+{% if alertmanager_receiver_type == 'discord' %}
+  alertmanager-discord:
+    image: rogerrum/alertmanager-discord:latest
+    environment:
+      DISCORD_WEBHOOK: "{{ alertmanager_discord_webhook_url }}"
+      DISCORD_USERNAME: "alertmanager"
+      LISTEN_ADDRESS: 0.0.0.0:9094
+    networks:
+      - monitoring
+    restart: unless-stopped
+    labels:
+      - com.centurylinklabs.watchtower.enable=true
+{% endif %}
+
+volumes:
+  alertmanager_data:
+
+networks:
+  monitoring:
+    external: true
+    name: monitoring