heartwood every commit a ring

Improve alert system with state tracking and recovery notifications

ca3dafdd by Isaac Bythewood · 11 months ago

Improve alert system with state tracking and recovery notifications

- Add alert_state field to track up/down status transitions
- Implement recovery email and Discord notifications
- Send alerts only on state changes (up->down, down->up)
- Require 2 consecutive failures before sending down alerts
- Add property_recovery.html email template
- Rename alert methods for clarity (send_email -> send_down_email)
added properties/migrations/0007_add_alert_state_tracking.py
@@ -0,0 +1,25 @@# Generated by Django 4.1 on 2025-05-31 20:08from django.db import migrations, modelsclass Migration(migrations.Migration):    dependencies = [        ("properties", "0006_property_crawler_insights"),    ]    operations = [        migrations.AddField(            model_name="property",            name="alert_state",            field=models.CharField(                choices=[("up", "Up"), ("down", "Down")], default="up", max_length=10            ),        ),        migrations.AddField(            model_name="property",            name="last_alert_sent",            field=models.DateTimeField(blank=True, null=True),        ),    ]
modified properties/models.py
@@ -108,7 +108,7 @@ class SecurityMixin:class AlertsMixin:    def send_email(self):    def send_down_email(self):        subject = f"Status: {self.name} is down!"        message = render_to_string("emails/property_down.html", {"property": self})        from_email = "noreply@bythewood.me"
@@ -117,27 +117,72 @@ class AlertsMixin:        email.content_subtype = "html"        email.send()    def send_discord_message(self):    def send_recovery_email(self):        subject = f"Status: {self.name} is back up!"        message = render_to_string("emails/property_recovery.html", {"property": self})        from_email = "noreply@bythewood.me"        to_emails = [self.user.email]        email = EmailMessage(subject, message, from_email, to_emails)        email.content_subtype = "html"        email.send()    def send_down_discord_message(self):        if self.user.discord_webhook_url:            payload = {                "username": "Status",                "embeds": [                    {                        "title": "Status",                        "title": "Status Alert",                        "description": f"{self.url} is down!",                        "color": 16711680,                        "color": 16711680,  # Red                        "timestamp": timezone.now().isoformat(),                    }                ],            }            requests.post(self.user.discord_webhook_url, json=payload)    def send_alerts(self):        # if the past two checks were != 200 send alerts        checks = self.statuses.order_by("-created_at")[:2]        if checks[0].status_code != 200 and checks[1].status_code != 200:            self.send_email()            self.send_discord_message()    def send_recovery_discord_message(self):        if self.user.discord_webhook_url:            payload = {                "username": "Status",                "embeds": [                    {                        "title": "Status Recovery",                        "description": f"{self.url} is back up!",                        "color": 65280,  # Green                        "timestamp": timezone.now().isoformat(),                    }                ],            }            requests.post(self.user.discord_webhook_url, json=payload)    def send_alerts(self, current_status_code):        """        Send alerts based on state transitions:        - Send 'down' alert when site goes from UP to DOWN        - Send 'recovery' alert when site goes from DOWN to UP        - No alerts for consecutive failures or consecutive successes        """        is_currently_up = current_status_code == 200        # Determine if we need to send an alert based on state change        if is_currently_up and self.alert_state == 'down':            # Site recovered: was down, now up            self.send_recovery_email()            self.send_recovery_discord_message()            self.alert_state = 'up'            self.last_alert_sent = timezone.now()            self.save(update_fields=['alert_state', 'last_alert_sent'])        elif not is_currently_up and self.alert_state == 'up':            # Site went down: was up, now down            # Only send if we have at least 2 consecutive failures to avoid false positives            checks = self.statuses.order_by("-created_at")[:2]            if len(checks) >= 2 and checks[0].status_code != 200 and checks[1].status_code != 200:                self.send_down_email()                self.send_down_discord_message()                self.alert_state = 'down'                self.last_alert_sent = timezone.now()                self.save(update_fields=['alert_state', 'last_alert_sent'])class CrawlerMixin:
@@ -317,6 +362,14 @@ class Property(CrawlerMixin, AlertsMixin, SecurityMixin, models.Model):    last_lighthouse_run_at = models.DateTimeField(blank=True, null=True)    next_lighthouse_run_at = models.DateTimeField(blank=True, null=True)    # Alert state tracking    last_alert_sent = models.DateTimeField(blank=True, null=True)    alert_state = models.CharField(        max_length=10,        choices=[('up', 'Up'), ('down', 'Down')],        default='up'    )    created_at = models.DateTimeField(auto_now_add=True)    updated_at = models.DateTimeField(auto_now=True)
@@ -376,8 +429,8 @@ class Property(CrawlerMixin, AlertsMixin, SecurityMixin, models.Model):    def process_check(self):        check = self.run_check()        if check.status_code != 200:            self.send_alerts()        # Always check for state changes, regardless of current status        self.send_alerts(check.status_code)    def get_next_run_at_lighthouse(self):        """
added properties/templates/emails/property_recovery.html
@@ -0,0 +1,157 @@<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"><html>  <head>    <meta http-equiv="x-ua-compatible" content="ie=edge">    <meta name="x-apple-disable-message-reformatting">    <meta name="viewport" content="width=device-width, initial-scale=1">    <meta name="format-detection" content="telephone=no, date=no, address=no, email=no">    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">    <style type="text/css">      body,table,td{font-family:Helvetica,Arial,sans-serif !important}.ExternalClass{width:100%}.ExternalClass,.ExternalClass p,.ExternalClass span,.ExternalClass font,.ExternalClass td,.ExternalClass div{line-height:150%}a{text-decoration:none}*{color:inherit}a[x-apple-data-detectors],u+#body a,#MessageViewBody a{color:inherit;text-decoration:none;font-size:inherit;font-family:inherit;font-weight:inherit;line-height:inherit}img{-ms-interpolation-mode:bicubic}table:not([class^=s-]){font-family:Helvetica,Arial,sans-serif;mso-table-lspace:0pt;mso-table-rspace:0pt;border-spacing:0px;border-collapse:collapse}table:not([class^=s-]) td{border-spacing:0px;border-collapse:collapse}@media screen and (max-width: 600px){.w-full,.w-full>tbody>tr>td{width:100% !important}.p-3:not(table),.p-3:not(.btn)>tbody>tr>td,.p-3.btn td a{padding:12px !important}*[class*=s-lg-]>tbody>tr>td{font-size:0 !important;line-height:0 !important;height:0 !important}.s-4>tbody>tr>td{font-size:16px !important;line-height:16px !important;height:16px !important}.s-10>tbody>tr>td{font-size:40px !important;line-height:40px !important;height:40px !important}}    </style>  </head>  <body class="bg-light" style="outline: 0; width: 100%; min-width: 100%; height: 100%; -webkit-text-size-adjust: 100%; -ms-text-size-adjust: 100%; font-family: Helvetica, Arial, sans-serif; line-height: 24px; font-weight: normal; font-size: 16px; -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; color: #000000; margin: 0; padding: 0; border-width: 0;" bgcolor="#f7fafc">    <table class="bg-light body" valign="top" role="presentation" border="0" cellpadding="0" cellspacing="0" style="outline: 0; width: 100%; min-width: 100%; height: 100%; -webkit-text-size-adjust: 100%; -ms-text-size-adjust: 100%; font-family: Helvetica, Arial, sans-serif; line-height: 24px; font-weight: normal; font-size: 16px; -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; color: #000000; margin: 0; padding: 0; border-width: 0;" bgcolor="#f7fafc">      <tbody>        <tr>          <td valign="top" style="line-height: 24px; font-size: 16px; margin: 0;" align="left" bgcolor="#f7fafc">            <table class="container" role="presentation" border="0" cellpadding="0" cellspacing="0" style="width: 100%;">              <tbody>                <tr>                  <td align="center" style="line-height: 24px; font-size: 16px; margin: 0; padding: 0 16px;">                    <!--[if (gte mso 9)|(IE)]>                      <table align="center" role="presentation">                        <tbody>                          <tr>                            <td width="600">                    <![endif]-->                    <table align="center" role="presentation" border="0" cellpadding="0" cellspacing="0" style="width: 100%; max-width: 600px; margin: 0 auto;">                      <tbody>                        <tr>                          <td style="line-height: 24px; font-size: 16px; margin: 0;" align="left">                            <table class="s-10 w-full" role="presentation" border="0" cellpadding="0" cellspacing="0" style="width: 100%;" width="100%">                              <tbody>                                <tr>                                  <td style="line-height: 40px; font-size: 40px; width: 100%; height: 40px; margin: 0;" align="left" width="100%" height="40">                                    &#160;                                  </td>                                </tr>                              </tbody>                            </table>                            <table class="card  rounded-none" role="presentation" border="0" cellpadding="0" cellspacing="0" style="border-radius: 0px; border-collapse: separate !important; width: 100%; overflow: hidden; border: 1px solid #e2e8f0;" bgcolor="#ffffff">                              <tbody>                                <tr>                                  <td style="line-height: 24px; font-size: 16px; width: 100%; border-radius: 0px; margin: 0;" align="left" bgcolor="#ffffff">                                    <table class="card-body" role="presentation" border="0" cellpadding="0" cellspacing="0" style="width: 100%;">                                      <tbody>                                        <tr>                                          <td style="line-height: 24px; font-size: 16px; width: 100%; margin: 0; padding: 20px;" align="left">                                            <h1 class="bg-success text-white" style="background-color: #28a745; color: #ffffff; padding-top: 0; padding-bottom: 0; font-weight: 500; vertical-align: baseline; font-size: 36px; line-height: 43.2px; margin: 0;" align="left">                                              <table class="p-3" role="presentation" border="0" cellpadding="0" cellspacing="0">                                                <tbody>                                                  <tr>                                                    <td style="line-height: 24px; font-size: 16px; margin: 0; padding: 12px;" align="left">                                                      <strong class="h2" style="padding-top: 0; padding-bottom: 0; font-weight: 500; text-align: left; vertical-align: baseline; font-size: 32px; line-height: 38.4px; margin: 0;">Property recovered</strong>                                                    </td>                                                  </tr>                                                </tbody>                                              </table>                                            </h1>                                            <table class="s-4 w-full" role="presentation" border="0" cellpadding="0" cellspacing="0" style="width: 100%;" width="100%">                                              <tbody>                                                <tr>                                                  <td style="line-height: 16px; font-size: 16px; width: 100%; height: 16px; margin: 0;" align="left" width="100%" height="16">                                                    &#160;                                                  </td>                                                </tr>                                              </tbody>                                            </table>                                            <table class="alert alert-success rounded-none" role="presentation" border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate !important; width: 100%; border-radius: 0px; border-width: 0;">                                              <tbody>                                                <tr>                                                  <td style="line-height: 24px; font-size: 16px; border-radius: 0px; color: #1e4d20; margin: 0; padding: 12px 20px; border: 1px solid transparent;" align="left" bgcolor="#e6ffed">                                                    <div>                                                      Good news! Your property is back online and responding normally.                                                    </div>                                                  </td>                                                </tr>                                              </tbody>                                            </table>                                            <table class="s-4 w-full" role="presentation" border="0" cellpadding="0" cellspacing="0" style="width: 100%;" width="100%">                                              <tbody>                                                <tr>                                                  <td style="line-height: 16px; font-size: 16px; width: 100%; height: 16px; margin: 0;" align="left" width="100%" height="16">                                                    &#160;                                                  </td>                                                </tr>                                              </tbody>                                            </table>                                            <div class="space-y-3">                                              <table class="table" border="0" cellpadding="0" cellspacing="0" style="width: 100%; max-width: 100%;">                                                <tbody>                                                  <tr>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">                                                      <strong>name</strong>                                                    </td>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">{{ property.name }}</td>                                                  </tr>                                                  <tr>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">                                                      <strong>url</strong>                                                    </td>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">                                                      <a href="{{ property.url }}" target="_blank" style="color: #0d6efd;">{{ property.url }}</a>                                                    </td>                                                  </tr>                                                  <tr>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">                                                      <strong>status</strong>                                                    </td>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">{{ property.current_status }}</td>                                                  </tr>                                                  <tr>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">                                                      <strong>avg. time</strong>                                                    </td>                                                    <td style="line-height: 24px; font-size: 16px; border-top-width: 1px; border-top-color: #e2e8f0; border-top-style: solid; margin: 0; padding: 12px;" align="left" valign="top">{{ property.avg_response_time }}</td>                                                  </tr>                                                </tbody>                                              </table>                                            </div>                                          </td>                                        </tr>                                      </tbody>                                    </table>                                  </td>                                </tr>                              </tbody>                            </table>                            <table class="s-10 w-full" role="presentation" border="0" cellpadding="0" cellspacing="0" style="width: 100%;" width="100%">                              <tbody>                                <tr>                                  <td style="line-height: 40px; font-size: 40px; width: 100%; height: 40px; margin: 0;" align="left" width="100%" height="40">                                    &#160;                                  </td>                                </tr>                              </tbody>                            </table>                          </td>                        </tr>                      </tbody>                    </table>                    <!--[if (gte mso 9)|(IE)]>                    </td>                  </tr>                </tbody>              </table>                    <![endif]-->                  </td>                </tr>              </tbody>            </table>          </td>        </tr>      </tbody>    </table>  </body></html>