diff --git a/README.md b/README.md index 8a47646b..07885898 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ Follow [these instructions](./docs/slack_app_create.md) to create a new Slack Ap | `INCIDENT_CHANNEL_ID` | When an incident is declared, a 'headline' post is sent to a central channel.

See the [demo app settings](./demo/demo/settings/dev.py) for an example of how to get the incident channel ID from the Slack API. | | `INCIDENT_BOT_ID` | We want to invite the Bot to all Incident Channels, so need to know its ID.

See the [demo app settings](./demo/demo/settings/dev.py) for an example of how to get the bot ID from the Slack API. | | `SLACK_CLIENT` | Response needs a shared global instance of a Slack Client to talk to the Slack API. Typically this does not require any additional configuration. 
from response.slack.client import SlackClient
SLACK_CLIENT = SlackClient(SLACK_TOKEN)
| +| `GRAFANA_URL`
`GRAFANA_TOKEN`| (OPTIONAL) Send annotations to grafana
See [grafana annotations support](./docs/grafana_annoations_support.md). | ## 3. Running the server diff --git a/demo/demo/settings/base.py b/demo/demo/settings/base.py index ebeb8deb..ccc8795b 100644 --- a/demo/demo/settings/base.py +++ b/demo/demo/settings/base.py @@ -15,6 +15,7 @@ from django.core.exceptions import ImproperlyConfigured +from response.grafana.client import GrafanaClient from response.slack.client import SlackClient logger = logging.getLogger(__name__) @@ -197,3 +198,8 @@ def get_env_var(setting, warn_only=False): # Whether to use https://pypi.org/project/bleach/ to strip potentially dangerous # HTML input in string fields RESPONSE_SANITIZE_USER_INPUT = True + +GRAFANA_URL = get_env_var("GRAFANA_URL", warn_only=True) +GRAFANA_CLIENT = None +if GRAFANA_URL: + GRAFANA_CLIENT = GrafanaClient(GRAFANA_URL, get_env_var("GRAFANA_TOKEN")) diff --git a/docs/grafana-1.png b/docs/grafana-1.png new file mode 100644 index 00000000..8589eb5e Binary files /dev/null and b/docs/grafana-1.png differ diff --git a/docs/grafana-2.png b/docs/grafana-2.png new file mode 100644 index 00000000..25d2861a Binary files /dev/null and b/docs/grafana-2.png differ diff --git a/docs/grafana-3.png b/docs/grafana-3.png new file mode 100644 index 00000000..e037c270 Binary files /dev/null and b/docs/grafana-3.png differ diff --git a/docs/grafana_annoations_support.md b/docs/grafana_annoations_support.md new file mode 100644 index 00000000..40eb235e --- /dev/null +++ b/docs/grafana_annoations_support.md @@ -0,0 +1,30 @@ +# Grafana annotation support + +

+ +

+ +## Respone configuration +To add an incident annotation on your diagrams (see exemple above), please set the following variables: + +* `GRAFANA_URL`: The URL of your grafana instance (ex: https://grafana.example.net); +* `GRAFANA_TOKEN`: A Grafana API key with Editor role. + +## Grafana configuration + +Edit your dashboard settings. + +Go to the "Annotations" section: + +![dashboard-edit](./grafana-1.png) + +Then add a new annotation with the following settings: + +* Name: Incident (for example) +* Data source: -- Grafana -- +* Color: Enabled :heavy_check_mark: (OPTIONAL) +* Filter by: Tags +* Tags: + * incident + +![annotations setup](./grafana-2.png) diff --git a/response/apps.py b/response/apps.py index 612f1325..2dfed74a 100644 --- a/response/apps.py +++ b/response/apps.py @@ -18,6 +18,8 @@ def ready(self): from .core import signals as core_signals # noqa: F401 + from .grafana import signals as grafana_signals # noqa: F401 + site_settings.RESPONSE_LOGIN_REQUIRED = getattr( site_settings, "RESPONSE_LOGIN_REQUIRED", True ) diff --git a/response/core/models/incident.py b/response/core/models/incident.py index 7edbbcb2..e7c8740d 100644 --- a/response/core/models/incident.py +++ b/response/core/models/incident.py @@ -77,6 +77,8 @@ class Incident(models.Model): max_length=10, blank=True, null=True, choices=SEVERITIES ) + grafana_annotation_id = models.PositiveIntegerField(null=True, blank=True) + def __str__(self): return self.report diff --git a/response/grafana/client.py b/response/grafana/client.py new file mode 100644 index 00000000..d772462f --- /dev/null +++ b/response/grafana/client.py @@ -0,0 +1,75 @@ +import logging + +import requests + +logger = logging.getLogger(__name__) + + +class GrafanaError(Exception): + def __init__(self, message, grafana_error=None): + self.message = message + self.grafana_error = grafana_error + + +class GrafanaClient(object): + def __init__(self, url, token): + self.url = url + self.token = token + + def create_annotation(self, **kwargs): + logger.info("Create Annotation") + + payload = { + "time": kwargs.get("time"), + "tags": kwargs.get("tags"), + "text": kwargs.get("text"), + } + + headers = { + "Authorization": "Bearer {}".format(self.token), + "Content-Type": "application/json", + } + res = requests.post( + "{0}/api/{1}".format(self.url, "annotations"), + headers=headers, + json=payload, + verify=True, + ) + + result = res.json() + res.close() + if res.status_code == 200: + return result + raise GrafanaError(f"Failed to create annotations '{result}'") + + def update_annotation(self, annotation_id, time, time_end, text, tags): + logger.info(f"Update Annotation: '{annotation_id}'") + + payload = {} + if time: + payload["time"] = int(time.timestamp() * 1000) + if time_end: + payload["timeEnd"] = int(time_end.timestamp() * 1000) + if text: + payload["text"] = text + if tags: + payload["tags"] = tags + + headers = { + "Authorization": "Bearer {}".format(self.token), + "Content-Type": "application/json", + } + res = requests.patch( + "{0}/api/{1}/{2}".format(self.url, "annotations", annotation_id), + headers=headers, + json=payload, + verify=True, + ) + + result = res.json() + res.close() + if res.status_code == 200: + return result + raise GrafanaError( + f"Failed to update annotation: '{annotation_id}': '{result}'" + ) diff --git a/response/grafana/signals.py b/response/grafana/signals.py new file mode 100644 index 00000000..25b3b6ef --- /dev/null +++ b/response/grafana/signals.py @@ -0,0 +1,52 @@ +import logging + +from django.conf import settings +from django.db.models.signals import post_save, pre_save +from django.dispatch import receiver + +from response.core.models import Incident + +logger = logging.getLogger(__name__) + + +@receiver(post_save, sender=Incident) +def update_grafana_annotation_after_incident_save(sender, instance: Incident, **kwargs): + """ + Reflect changes to incidents in the grafana annotation + + Important: this is called in the synchronous /incident flow so must remain fast (<2 secs) + + """ + if settings.GRAFANA_CLIENT and instance.grafana_annotation_id: + tags = ["incident", instance.severity_text()] + text = f"{instance.report} \n {instance.summary}" + + settings.GRAFANA_CLIENT.update_annotation( + instance.grafana_annotation_id, + time=instance.report_time, + time_end=instance.end_time, + text=text, + tags=tags, + ) + + +@receiver(pre_save, sender=Incident) +def create_grafana_annotation_before_incident_save( + sender, instance: Incident, **kwargs +): + """ + Create a grafana annotation ticket before saving the incident + + Important: this is called in the synchronous /incident flow so must remain fast (<2 secs) + + """ + + if settings.GRAFANA_CLIENT and not instance.grafana_annotation_id: + tags = ["incident", instance.severity_text()] + text = f"{instance.report} \n {instance.summary}" + start_time = int(instance.report_time.timestamp() * 1000) + + grafana_annotation = settings.GRAFANA_CLIENT.create_annotation( + time=start_time, tags=tags, text=text + ) + instance.grafana_annotation_id = grafana_annotation["id"] diff --git a/response/migrations/0018_incident_grafana_annotation_id.py b/response/migrations/0018_incident_grafana_annotation_id.py new file mode 100644 index 00000000..2ca489fa --- /dev/null +++ b/response/migrations/0018_incident_grafana_annotation_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.20 on 2021-04-28 09:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("response", "0017_externaluser_deleted"), + ] + + operations = [ + migrations.AddField( + model_name="incident", + name="grafana_annotation_id", + field=models.PositiveIntegerField(blank=True, null=True), + ), + ]