Speed up dashboard, split property view, surface bot traffic

@@ -1,6 +1,7 @@import uuidfrom django.db import modelsfrom django.db.models import Count, Qfrom django.contrib.auth.models import AbstractUser

@@ -10,27 +11,29 @@ class User(AbstractUser):    def __str__(self):        return self.username    def _event_totals(self):        # One query rolls up everything the profile page displays.        if not hasattr(self, "_cached_event_totals"):            self._cached_event_totals = self.properties.aggregate(                total_properties=Count("id", distinct=True),                total_events=Count("events"),                total_page_views=Count("events", filter=Q(events__event="page_view")),                total_session_starts=Count("events", filter=Q(events__event="session_start")),            )        return self._cached_event_totals    @property    def total_properties(self):        return self.properties.count()        return self._event_totals()["total_properties"]    @property    def total_events(self):        total_events = 0        for property in self.properties.all():            total_events += property.total_events        return total_events        return self._event_totals()["total_events"]    @property    def total_page_views(self):        total_page_views = 0        for property in self.properties.all():            total_page_views += property.total_page_views        return total_page_views        return self._event_totals()["total_page_views"]    @property    def total_session_starts(self):        total_session_starts = 0        for property in self.properties.all():            total_session_starts += property.total_session_starts        return total_session_starts        return self._event_totals()["total_session_starts"]

@@ -108,15 +108,10 @@ def collect(request):            event_obj.data['device'] = 'Tablet'        else:            event_obj.data['device'] = 'Desktop'        if not ua.is_bot:            # I've decided I don't want to save bots in the database but you            # are free to change this!            event_obj.save()    else:        # If we don't have a user agent let's save just in case because it might        # be a server side event or the latest chrome which sometimes doesn't        # have one. We do try to get userAgentData in our collector.js too which        # gets auto set into the correct data attributes.        event_obj.save()        if ua.is_bot:            event_obj.data['is_bot'] = True            event_obj.data['bot_name'] = ua.browser.family or 'Unknown bot'    event_obj.save()    return HttpResponse(status=204)

@@ -0,0 +1,62 @@BUILT_IN_EVENTS = ["session_start", "page_view", "page_leave", "click", "scroll"]US_STATES = {    "Alabama": "AL",    "Alaska": "AK",    "Arizona": "AZ",    "Arkansas": "AR",    "California": "CA",    "Colorado": "CO",    "Connecticut": "CT",    "Delaware": "DE",    "Florida": "FL",    "Georgia": "GA",    "Hawaii": "HI",    "Idaho": "ID",    "Illinois": "IL",    "Indiana": "IN",    "Iowa": "IA",    "Kansas": "KS",    "Kentucky": "KY",    "Louisiana": "LA",    "Maine": "ME",    "Maryland": "MD",    "Massachusetts": "MA",    "Michigan": "MI",    "Minnesota": "MN",    "Mississippi": "MS",    "Missouri": "MO",    "Montana": "MT",    "Nebraska": "NE",    "Nevada": "NV",    "New Hampshire": "NH",    "New Jersey": "NJ",    "New Mexico": "NM",    "New York": "NY",    "North Carolina": "NC",    "North Dakota": "ND",    "Ohio": "OH",    "Oklahoma": "OK",    "Oregon": "OR",    "Pennsylvania": "PA",    "Rhode Island": "RI",    "South Carolina": "SC",    "South Dakota": "SD",    "Tennessee": "TN",    "Texas": "TX",    "Utah": "UT",    "Vermont": "VT",    "Virginia": "VA",    "Washington": "WA",    "West Virginia": "WV",    "Wisconsin": "WI",    "Wyoming": "WY",    "District of Columbia": "DC",    "American Samoa": "AS",    "Guam": "GU",    "Northern Mariana Islands": "MP",    "Puerto Rico": "PR",    "United States Minor Outlying Islands": "UM",    "U.S. Virgin Islands": "VI",}

@@ -0,0 +1,21 @@# Generated by Django 6.0.4 on 2026-04-17 03:58from django.db import migrations, modelsclass Migration(migrations.Migration):    dependencies = [        ('properties', '0003_alter_property_custom_cards'),    ]    operations = [        migrations.AddIndex(            model_name='event',            index=models.Index(fields=['property', 'created_at'], name='properties__propert_9b5576_idx'),        ),        migrations.AddIndex(            model_name='event',            index=models.Index(fields=['property', 'event', 'created_at'], name='properties__propert_244021_idx'),        ),    ]

@@ -85,4 +85,6 @@ class Event(models.Model):    class Meta:        indexes = [            models.Index(fields=['created_at']),            models.Index(fields=['property', 'created_at']),            models.Index(fields=['property', 'event', 'created_at']),        ]

@@ -1,19 +1,27 @@from django.db import modelsfrom django.db.models.functions import Castfrom django.db.models import Avg, Count, FloatField, Qfrom django.db.models.functions import Cast, TruncDatefrom django.utils import timezonefrom .constants import BUILT_IN_EVENTS, US_STATESdef total_live_users(property_obj):def human_events(events):    """Exclude bot-tagged events from a queryset.    SQLite's JSON NOT-EQUAL doesn't match rows where the key is missing, so    `exclude(data__is_bot=True)` drops everything. `has_key` matches only rows    that contain the key — and we only ever set is_bot when the UA is a bot.    """    Returns the total number of live users for a given property within the last    30 minutes.    return events.exclude(data__has_key="is_bot")    :param property_obj: Property objectdef total_live_users(property_obj):    """    Total unique user_ids seen in the last 30 minutes.    """    return (        property_obj.events.filter(            created_at__gte=timezone.now() - timezone.timedelta(minutes=30)        )        human_events(property_obj.events)        .filter(created_at__gte=timezone.now() - timezone.timedelta(minutes=30))        .exclude(data__user_id__isnull=True)        .values("data__user_id")        .distinct()

@@ -21,161 +29,322 @@ def total_live_users(property_obj):    )def standard_event_cards(events_filtered, events_filtered_prev):    """    Returns the standard event cards for a given property.# Cap time-on-page to filter out idle-tab outliers (left open for hours).# < 1s is likely bot/instant-exit; > 30min is almost certainly idle.TIME_ON_PAGE_MIN_S = 1TIME_ON_PAGE_MAX_S = 30 * 60    :param events_filtered: Filtered events    :param events_filtered_prev: Filtered events from previous period    :return: Event cards array    """    event_cards = []    total_session_starts = events_filtered.filter(event="session_start").count()    total_session_starts_prev = events_filtered_prev.filter(event="session_start").count()    event_cards.append({        "name": "Total session starts",        "value": total_session_starts,        "percent_change": round((total_session_starts - total_session_starts_prev) / total_session_starts_prev * 100) if total_session_starts_prev > 0 else 0,        "help_text": "Unique users visiting your site for your selected date range.",    })    total_page_views = events_filtered.filter(event="page_view").count()    total_page_views_prev = events_filtered_prev.filter(event="page_view").count()    event_cards.append({        "name": "Total page views",        "value": total_page_views,        "percent_change": round((total_page_views - total_page_views_prev) / total_page_views_prev * 100) if total_page_views_prev else 0,        "help_text": "Total pages viewed for your selected date range.",    })def _pct_change(current, previous):    if not previous:        return 0    return round((current - previous) / previous * 100)    total_clicks = events_filtered.filter(event="click").count()    total_clicks_prev = events_filtered_prev.filter(event="click").count()    event_cards.append({        "name": "Total clicks",        "value": total_clicks,        "percent_change": round((total_clicks - total_clicks_prev) / total_clicks_prev * 100) if total_clicks_prev > 0 else 0,        "help_text": "Total clicks users made on all your pages for your selected date range.",    })    total_scrolls = events_filtered.filter(event="scroll").count()    total_scrolls_prev = events_filtered_prev.filter(event="scroll").count()    event_cards.append({        "name": "Total scrolls",        "value": total_scrolls,        "percent_change": round((total_scrolls - total_scrolls_prev) / total_scrolls_prev * 100) if total_scrolls_prev > 0 else 0,        "help_text": "Total scrolls users made on all your pages for your selected date range.",    })def _event_counts(qs):    """Single-aggregate pass for the five built-in event counts + total."""    return qs.aggregate(        session_start=Count("id", filter=Q(event="session_start")),        page_view=Count("id", filter=Q(event="page_view")),        click=Count("id", filter=Q(event="click")),        scroll=Count("id", filter=Q(event="scroll")),        total=Count("id"),    )    total_events = events_filtered.count()    total_events_prev = events_filtered_prev.count()    event_cards.append({        "name": "Total events",        "value": total_events,        "percent_change": round((total_events - total_events_prev) / total_events_prev * 100) if total_events_prev else 0,        "help_text": "All events for your selected date range, including custom events.",    })def _engaged_users(qs, session_starts):    if not session_starts:        return 0    engaged = (        qs.exclude(data__user_id__isnull=True)        .values("data__user_id")        .annotate(c=Count("id"))        .filter(c__gte=10)        .count()    )    return round(engaged / session_starts * 100, 2)def _avg_time_on_page(qs):    try:        total_unique_users_with_events = (            events_filtered.exclude(data__user_id__isnull=True)            .values("data__user_id")            .distinct()            .annotate(count=models.Count("data__user_id"))            .filter(count__gte=10)            .count()        )        total_user_engagement = round(            total_unique_users_with_events / total_session_starts * 100, 2        )    except ZeroDivisionError:        total_user_engagement = 0    try:        total_unique_users_with_events_prev = (            events_filtered_prev.exclude(data__user_id__isnull=True)            .values("data__user_id")            .distinct()            .annotate(count=models.Count("data__user_id"))            .filter(count__gte=10)            .count()        )        total_user_engagement_prev = round(            total_unique_users_with_events_prev / total_session_starts_prev * 100, 2        avg = (            qs.filter(event="page_leave")            .annotate(time_on_page_s=Cast("data__time_on_page", FloatField()) / 1000)            .filter(                time_on_page_s__gte=TIME_ON_PAGE_MIN_S,                time_on_page_s__lte=TIME_ON_PAGE_MAX_S,            )            .aggregate(avg=Avg("time_on_page_s"))["avg"]        )    except ZeroDivisionError:        total_user_engagement_prev = 0    event_cards.append({        return round(avg, 2) if avg is not None else 0    except TypeError:        return 0def standard_event_cards(events_filtered, events_filtered_prev):    """Standard metric cards. Two aggregate queries plus engagement/time-on-page helpers."""    cur = _event_counts(events_filtered)    prev = _event_counts(events_filtered_prev)    cards = [        {            "name": "Total session starts",            "value": cur["session_start"],            "percent_change": _pct_change(cur["session_start"], prev["session_start"]),            "help_text": "Unique users visiting your site for your selected date range.",        },        {            "name": "Total page views",            "value": cur["page_view"],            "percent_change": _pct_change(cur["page_view"], prev["page_view"]),            "help_text": "Total pages viewed for your selected date range.",        },        {            "name": "Total clicks",            "value": cur["click"],            "percent_change": _pct_change(cur["click"], prev["click"]),            "help_text": "Total clicks users made on all your pages for your selected date range.",        },        {            "name": "Total scrolls",            "value": cur["scroll"],            "percent_change": _pct_change(cur["scroll"], prev["scroll"]),            "help_text": "Total scrolls users made on all your pages for your selected date range.",        },        {            "name": "Total events",            "value": cur["total"],            "percent_change": _pct_change(cur["total"], prev["total"]),            "help_text": "All events for your selected date range, including custom events.",        },    ]    engagement_cur = _engaged_users(events_filtered, cur["session_start"])    engagement_prev = _engaged_users(events_filtered_prev, prev["session_start"])    cards.append({        "name": "Total user engagement",        "value": f"{total_user_engagement}%",        "percent_change": round((total_user_engagement - total_user_engagement_prev) / total_user_engagement_prev * 100) if total_user_engagement_prev else 0,        "value": f"{engagement_cur}%",        "percent_change": _pct_change(engagement_cur, engagement_prev),        "help_text": "An engaged user is a user more than 10 events collected for your selected date range.",    })    # Cap time-on-page to filter out idle-tab outliers (left open for hours).    # Anything < 1s is likely bot/instant-exit; anything > 30min is almost    # certainly an idle/abandoned tab rather than real engagement.    TIME_ON_PAGE_MIN_S = 1    TIME_ON_PAGE_MAX_S = 30 * 60    def _avg_time_on_page(qs):        try:            avg = qs.filter(event="page_leave").annotate(                time_on_page_s=Cast("data__time_on_page", models.FloatField()) / 1000            ).filter(                time_on_page_s__gte=TIME_ON_PAGE_MIN_S,                time_on_page_s__lte=TIME_ON_PAGE_MAX_S,            ).aggregate(avg=models.Avg("time_on_page_s"))["avg"]            return round(avg, 2) if avg is not None else 0        except TypeError:            return 0    avg_time_on_page = _avg_time_on_page(events_filtered)    avg_time_on_page_prev = _avg_time_on_page(events_filtered_prev)    event_cards.append({    time_cur = _avg_time_on_page(events_filtered)    time_prev = _avg_time_on_page(events_filtered_prev)    cards.append({        "name": "Avg. time on page",        "value": f"{avg_time_on_page}s",        "percent_change": round((avg_time_on_page - avg_time_on_page_prev) / avg_time_on_page_prev * 100) if avg_time_on_page_prev else 0,        "value": f"{time_cur}s",        "percent_change": _pct_change(time_cur, time_prev),        "help_text": "Average time a user spends on each page. Sessions over 30 minutes are excluded as idle.",    })    return event_cards    return cardsdef custom_event_cards(property_obj, events_filtered, events_filtered_prev):    """    Returns the custom event cards for a given property.    """Returns (cards, custom_events). Custom events are non-built-in event names."""    custom_events = list(        human_events(property_obj.events)        .exclude(event__in=BUILT_IN_EVENTS)        .values("event")        .distinct()        .order_by("event")    )    :param events_filtered: Filtered events    :param events_filtered_prev: Filtered events from previous period    :return: A list of custom events and a list of custom event cards    """    event_cards = []    custom_events = property_obj.events.exclude(        event__in=["session_start", "page_view", "page_leave", "click", "scroll"]    ).values("event").distinct().order_by("event")    active_cards = []    for card in property_obj.custom_cards:        if card['value'] is True:            active_cards.append(card['event'])    # add active = True or active = False to custom_events if in active_cards    for card in custom_events:        if card['event'] in active_cards:            card['active'] = True        else:            card['active'] = False    for custom_event in custom_events:        if custom_event["event"] not in active_cards:    active_names = {c["event"] for c in property_obj.custom_cards if c.get("value") is True}    for ce in custom_events:        ce["active"] = ce["event"] in active_names    if not active_names:        return [], custom_events    cur = events_filtered.filter(event__in=active_names).values("event").annotate(c=Count("id"))    prev = events_filtered_prev.filter(event__in=active_names).values("event").annotate(c=Count("id"))    cur_map = {r["event"]: r["c"] for r in cur}    prev_map = {r["event"]: r["c"] for r in prev}    cards = []    for ce in custom_events:        if ce["event"] not in active_names:            continue        total_events = events_filtered.filter(event=custom_event["event"]).count()        total_events_prev = events_filtered_prev.filter(event=custom_event["event"]).count()        event_cards.append({            "name": custom_event["event"],            "value": total_events,            "percent_change": round((total_events - total_events_prev) / total_events_prev * 100) if total_events_prev else 0,        v = cur_map.get(ce["event"], 0)        p = prev_map.get(ce["event"], 0)        cards.append({            "name": ce["event"],            "value": v,            "percent_change": _pct_change(v, p),        })    return cards, custom_events    return event_cards, custom_eventsdef events_graph(events_filtered, date_end_obj, date_range):    """    One GROUP BY query for daily counts, then bucket into days/weeks/months    in Python. Buckets step backwards from date_end.    """    rows = (        events_filtered.annotate(day=TruncDate("created_at"))        .values("day")        .annotate(count=Count("id"))    )    by_day = {r["day"]: r["count"] for r in rows if r["day"]}    end_date = date_end_obj.date()    def bucket_sum(start_date, days):        return sum(            by_day.get(start_date + timezone.timedelta(days=j), 0)            for j in range(days)        )    if date_range <= 28:        points = [            {                "label": end_date - timezone.timedelta(days=i),                "count": by_day.get(end_date - timezone.timedelta(days=i), 0),            }            for i in range(date_range)        ]    elif date_range <= 6 * 28:        points = [            {                "label": end_date - timezone.timedelta(days=7 * w),                "count": bucket_sum(end_date - timezone.timedelta(days=7 * w), 7),            }            for w in range(date_range // 7)        ]    else:        points = [            {                "label": end_date - timezone.timedelta(days=28 * m),                "count": bucket_sum(end_date - timezone.timedelta(days=28 * m), 28),            }            for m in range(date_range // 28)        ]    points.sort(key=lambda k: k["label"])    for p in points:        p["label"] = p["label"].strftime("%b %-d")    return pointsdef _top_by_key(qs, key, limit=10, event=None):    """Generic top-N list for a JSON key with count."""    if event is not None:        qs = qs.filter(event=event)    rows = (        qs.exclude(**{f"{key}__isnull": True})        .exclude(**{key: ""})        .values(key)        .annotate(count=Count("id"))        .order_by("-count")[:limit]    )    return [{"label": r[key], "count": r["count"]} for r in rows]def events_by_screen_size(events_filtered, limit=7):    rows = (        events_filtered.filter(event="session_start")        .exclude(data__screen_width__isnull=True)        .values("data__screen_width", "data__screen_height")        .annotate(count=Count("id"))        .order_by("-count")[:limit]    )    return [        {            "label": f"{r['data__screen_width']}x{r['data__screen_height']}",            "count": r["count"],        }        for r in rows    ]def events_by_device(events_filtered, limit=7):    return _top_by_key(events_filtered, "data__device", limit, event="session_start")def events_by_browser(events_filtered, limit=7):    return _top_by_key(events_filtered, "data__browser", limit, event="session_start")def events_by_platform(events_filtered, limit=7):    return _top_by_key(events_filtered, "data__platform", limit, event="session_start")def events_by_page_url(events_filtered, limit=10):    return _top_by_key(events_filtered, "data__url", limit)def page_views_by_page_url(events_filtered, limit=10):    return _top_by_key(events_filtered, "data__url", limit, event="page_view")def events_by_custom_event(events_filtered, limit=10):    rows = (        events_filtered.exclude(event__in=BUILT_IN_EVENTS)        .values("event")        .annotate(count=Count("id"))        .order_by("-count")[:limit]    )    return [{"label": r["event"], "count": r["count"]} for r in rows]def session_starts_by_referrer(events_filtered, limit=10):    return _top_by_key(events_filtered, "data__referrer", limit, event="session_start")def page_views_by_utm(events_filtered, field, limit=10):    return _top_by_key(events_filtered, f"data__utm_{field}", limit, event="page_view")def session_starts_by_region(events_filtered, limit=10):    rows = (        events_filtered.filter(event="session_start")        .exclude(data__region__isnull=True)        .values("data__region")        .annotate(count=Count("id"))        .order_by("-count")[:limit]    )    return [{"label": r["data__region"], "count": r["count"]} for r in rows]def region_map_data(regions):    """Convert a regions list into the datamaps dict keyed by US state code."""    chart = {}    state_codes = set(US_STATES.values())    for r in regions:        label = r["label"]        if label in US_STATES:            chart[US_STATES[label]] = {"numberOfThings": r["count"]}        elif label in state_codes:            chart[label] = {"numberOfThings": r["count"]}    return chartdef bot_traffic(events_all, limit=10):    """    Bot-only stats for the dashboard's bot card. Takes the unfiltered (bots    included) events queryset.    """    bots = events_all.filter(data__is_bot=True)    total = bots.count()    if not total:        return {"total": 0, "top_bots": [], "top_pages": []}    top_bots = list(        bots.exclude(data__bot_name__isnull=True)        .exclude(data__bot_name="")        .values("data__bot_name")        .annotate(count=Count("id"))        .order_by("-count")[:limit]    )    top_pages = list(        bots.exclude(data__url__isnull=True)        .exclude(data__url="")        .values("data__url")        .annotate(count=Count("id"))        .order_by("-count")[:limit]    )    return {        "total": total,        "top_bots": [{"label": r["data__bot_name"], "count": r["count"]} for r in top_bots],        "top_pages": [{"label": r["data__url"], "count": r["count"]} for r in top_pages],    }

@@ -364,6 +364,58 @@  </div></div><div class="container my-4">  <div class="section-label mb-2">bot traffic · excluded from metrics above</div>  {% if bot_traffic.total > 0 %}  <div class="row g-3">    <div class="col-12 col-md-4">      <div class="metric-tile">        <div class="metric-label">Total bot events</div>        <div class="metric-value">{{ bot_traffic.total }}</div>      </div>    </div>    {% if bot_traffic.top_bots|length > 0 %}    <div class="col-12 col-md-4">      <div class="rank-list">        <div class="rank-list-header">          <span class="rank-list-title">top bots</span>          <span class="rank-list-col">events</span>        </div>        <div class="rank-list-body">          {% for item in bot_traffic.top_bots %}          <div class="rank-list-row">            <span class="rank-label">{{ item.label }}</span>            <span class="rank-count">{{ item.count }}</span>          </div>          {% endfor %}        </div>      </div>    </div>    {% endif %}    {% if bot_traffic.top_pages|length > 0 %}    <div class="col-12 col-md-4">      <div class="rank-list">        <div class="rank-list-header">          <span class="rank-list-title">top pages · bot hits</span>          <span class="rank-list-col">events</span>        </div>        <div class="rank-list-body">          {% for item in bot_traffic.top_pages %}          <div class="rank-list-row">            <span class="rank-label">{{ item.label }}</span>            <span class="rank-count">{{ item.count }}</span>          </div>          {% endfor %}        </div>      </div>    </div>    {% endif %}  </div>  {% else %}  <div class="text-muted small">No bot events in this range.</div>  {% endif %}</div><div class="modal fade" id="siteTagModal" tabindex="-1" aria-labelledby="siteTagModalLabel" aria-hidden="true">  <div class="modal-dialog modal-lg">    <div class="modal-content">

@@ -107,6 +107,24 @@{% for item in total_page_views_by_utm_campaign %}| `{{ item.label }}` | {{ item.count }} |{% endfor %}{% endif %}{% if bot_traffic.total %}## Bot traffic (excluded from metrics above)**Total bot events:** {{ bot_traffic.total }}{% if bot_traffic.top_bots %}| Bot | Events ||---|---|{% for item in bot_traffic.top_bots %}| {{ item.label }} | {{ item.count }} |{% endfor %}{% endif %}{% if bot_traffic.top_pages %}### Top pages hit by bots| URL | Events ||---|---|{% for item in bot_traffic.top_pages %}| `{{ item.label }}` | {{ item.count }} |{% endfor %}{% endif %}{% endif %}---_Report generated by Analytics — self-hosted website analytics._

@@ -3,8 +3,8 @@ import uuidfrom django.conf import settingsfrom django.contrib import messagesfrom django.core.cache import cachefrom django.core.files.storage import default_storagefrom django.db import modelsfrom django.http import HttpResponse, JsonResponsefrom django.shortcuts import redirect, renderfrom django.template.loader import render_to_string

@@ -17,6 +17,9 @@ from .forms import PropertyFormfrom .models import PropertyDASHBOARD_CACHE_TTL = 300  # secondsdef properties(request):    if not request.user.is_authenticated:        return redirect("/")

@@ -33,9 +36,9 @@ def properties(request):        form = PropertyForm()    properties = request.user.properties.all()    q = request.GET.get("q", None)    if q:        properties = properties.filter(name__icontains=q)    search = request.GET.get("q", None)    if search:        properties = properties.filter(name__icontains=search)    return render(        request,

@@ -45,7 +48,7 @@ def properties(request):            "title": "Properties",            "description": "Manage your properties.",            "properties": properties,            "q": q,            "q": search,        },    )

@@ -65,9 +68,6 @@ def property_delete(request, property_id):def adjust_custom_event_cards(request, property_id):    """    Adds and removes custom event cards on a property    """    if not request.user.is_authenticated:        return redirect("/")

@@ -80,16 +80,12 @@ def adjust_custom_event_cards(request, property_id):        custom_cards = json.loads(request.body.decode("utf-8"))        property_obj.custom_cards = custom_cards        property_obj.save()        print(property_obj.custom_cards)        return JsonResponse({"success": True})    return JsonResponse({"success": False})def adjust_is_public_property(request, property_id):    """    Sets the property to public or private    """    if not request.user.is_authenticated:        return redirect("/")

@@ -106,25 +102,66 @@ def adjust_is_public_property(request, property_id):    return JsonResponse({"success": False})def property(request, property_id):    context = {}def _dashboard_context(property_obj, date_start_obj, date_end_obj, date_range, filter_url):    """    Heavy context built from DB queries. Cacheable — no request-specific state.    """    events_all = property_obj.events.filter(        created_at__gte=date_start_obj, created_at__lte=date_end_obj    )    if filter_url:        events_all = events_all.filter(data__url=filter_url)    events_filtered = q.human_events(events_all)    # Get the property and check permissions    date_start_obj_prev = date_start_obj - timezone.timedelta(days=date_range)    date_end_obj_prev = date_end_obj - timezone.timedelta(days=date_range)    events_filtered_prev = q.human_events(        property_obj.events.filter(            created_at__gte=date_start_obj_prev, created_at__lte=date_end_obj_prev        )    )    if filter_url:        events_filtered_prev = events_filtered_prev.filter(data__url=filter_url)    event_cards = list(q.standard_event_cards(events_filtered, events_filtered_prev))    custom_cards, custom_events = q.custom_event_cards(        property_obj, events_filtered, events_filtered_prev    )    event_cards.extend(custom_cards)    regions = q.session_starts_by_region(events_filtered, limit=100)    return {        "event_cards": event_cards,        "custom_events": custom_events,        "total_events_graph": q.events_graph(events_filtered, date_end_obj, date_range),        "total_events_by_screen_size": q.events_by_screen_size(events_filtered),        "total_events_by_device": q.events_by_device(events_filtered),        "total_events_by_browser": q.events_by_browser(events_filtered),        "total_events_by_platform": q.events_by_platform(events_filtered),        "total_events_by_page_url": q.events_by_page_url(events_filtered),        "total_page_views_by_page_url": q.page_views_by_page_url(events_filtered),        "total_events_by_custom_event": q.events_by_custom_event(events_filtered),        "total_session_starts_by_referrer": q.session_starts_by_referrer(events_filtered),        "total_page_views_by_utm_medium": q.page_views_by_utm(events_filtered, "medium"),        "total_page_views_by_utm_source": q.page_views_by_utm(events_filtered, "source"),        "total_page_views_by_utm_campaign": q.page_views_by_utm(events_filtered, "campaign"),        "total_session_starts_by_region": regions[:10],        "total_session_starts_by_region_chart_data": q.region_map_data(regions),        "bot_traffic": q.bot_traffic(events_all),    }def property(request, property_id):    try:        property_obj = Property.objects.get(pk=property_id)        context["property"] = property_obj    except Property.DoesNotExist:        return redirect("properties")    if not property_obj.is_public and property_obj.user != request.user:        return redirect("properties")    # Set some basic page context variables    context["title"] = property_obj.name    context["description"] = "Analytics for " + property_obj.name    context["BASE_URL"] = settings.BASE_URL    # Date range filter which defaults to 28 days if nothing is selected    date_start = request.GET.get(        "date_start",        (timezone.now() - timezone.timedelta(days=28)).strftime("%Y-%m-%d"),

@@ -132,410 +169,52 @@ def property(request, property_id):    date_end = request.GET.get("date_end", timezone.now().strftime("%Y-%m-%d"))    date_range = request.GET.get("date_range", 28)    context["date_start"] = date_start    context["date_end"] = date_end    context["date_range"] = date_range    date_start_obj = timezone.datetime.strptime(date_start, "%Y-%m-%d")    date_end_obj = timezone.datetime.strptime(        date_end, "%Y-%m-%d"    ) + timezone.timedelta(hours=23, minutes=59, seconds=59)    # Set the timezone    date_start_obj = timezone.make_aware(        date_start_obj, timezone.get_current_timezone()        timezone.datetime.strptime(date_start, "%Y-%m-%d"),        timezone.get_current_timezone(),    )    date_end_obj = timezone.make_aware(        timezone.datetime.strptime(date_end, "%Y-%m-%d")        + timezone.timedelta(hours=23, minutes=59, seconds=59),        timezone.get_current_timezone(),    )    date_end_obj = timezone.make_aware(date_end_obj, timezone.get_current_timezone())    if date_range == "custom":        date_range = (date_end_obj - date_start_obj).days    else:        date_range = int(date_range)    # Get the current period based on the date range    events_filtered = property_obj.events.filter(        created_at__gte=date_start_obj, created_at__lte=date_end_obj    )    # Get the filter_url and filter by data__url if filter_url exists    filter_url = request.GET.get("filter_url", None)    if filter_url:        events_filtered = events_filtered.filter(data__url=filter_url)        context['filter_url'] = filter_url    # Get the previous period as well for comparisons    date_start_obj_prev = date_start_obj - timezone.timedelta(days=date_range)    date_end_obj_prev = date_end_obj - timezone.timedelta(days=date_range)    events_filtered_prev = property_obj.events.filter(        created_at__gte=date_start_obj_prev, created_at__lte=date_end_obj_prev    )    # Start querying our data    context["total_live_users"] = q.total_live_users(property_obj)    event_cards = []    event_cards.extend(q.standard_event_cards(events_filtered, events_filtered_prev))    custom_event_cards, custom_events = q.custom_event_cards(        property_obj, events_filtered, events_filtered_prev    # Heavy DB work goes through a 5-min cache. Live users stays uncached so    # "live" actually means live. ?report bypasses the cache so exports match    # whatever the user sees in the dashboard right now.    # Include updated_at so custom-cards/visibility changes bust the cache.    ver = int(property_obj.updated_at.timestamp())    cache_key = (        f"dash:{property_obj.id}:{ver}:{date_start}:{date_end}:{date_range}:{filter_url or ''}"    )    event_cards.extend(custom_event_cards)    context["custom_events"] = custom_events    context["event_cards"] = event_cards    #    # Total events per day graph    #    # If less than 28 days show each day, if less than 6 months then show    # each week. If less than 24 months then show each month. If more than 24    # months then show each year.    if date_range <= 28:        total_events_by_day = []        for day in range(date_range):            date = date_end_obj - timezone.timedelta(days=day)            count = events_filtered.filter(created_at__date=date).count()            total_events_by_day.append({"label": date, "count": count})        context["total_events_graph"] = sorted(            total_events_by_day, key=lambda k: k["label"]        )    elif date_range <= 6 * 28:        total_events_by_week = []        # group weeks sunday through saturday        for week in range(date_range // 7):            date = date_end_obj - timezone.timedelta(days=7 * week)            count = events_filtered.filter(                created_at__gte=date, created_at__lte=date + timezone.timedelta(days=6)            ).count()            total_events_by_week.append({"label": date, "count": count})        context["total_events_graph"] = sorted(            total_events_by_week, key=lambda k: k["label"]        )    if "report" in request.GET:        dashboard = _dashboard_context(property_obj, date_start_obj, date_end_obj, date_range, filter_url)    else:        total_events_by_month = []        # group months 1 through 31        for month in range(date_range // 28):            date = date_end_obj - timezone.timedelta(days=28 * month)            count = events_filtered.filter(                created_at__gte=date, created_at__lte=date + timezone.timedelta(days=27)            ).count()            total_events_by_month.append({"label": date, "count": count})        context["total_events_graph"] = sorted(            total_events_by_month, key=lambda k: k["label"]        )    for day in context["total_events_graph"]:        day["label"] = day["label"].strftime("%b %-d")    #    # Total events by screen size graph    #    total_events_by_screen_size = []    for event in (        events_filtered.filter(event="session_start")        .exclude(data__screen_width__isnull=True)        .values("data__screen_width", "data__screen_height")        .annotate(count=models.Count("data__screen_width"))        .order_by("-count")[:10]    ):        total_events_by_screen_size.append(            {                "label": str(event["data__screen_width"])                + "x"                + str(event["data__screen_height"]),                "count": event["count"],            }        )    context["total_events_by_screen_size"] = total_events_by_screen_size[:7]    #    # Total events by device graph    #    total_events_by_device = []    for event in (        events_filtered.filter(event="session_start")        .exclude(data__device__isnull=True)        .values("data__device")        .annotate(count=models.Count("data__device"))        .order_by("-count")[:10]    ):        if event["data__device"] != "" and event["data__device"] is not None:            total_events_by_device.append(                {"label": event["data__device"], "count": event["count"]}            )    context["total_events_by_device"] = total_events_by_device[:7]    #    # Total events by browser graph    #    total_events_by_browser = []    for event in (        events_filtered.filter(event="session_start")        .exclude(data__browser__isnull=True)        .values("data__browser")        .annotate(count=models.Count("data__browser"))        .order_by("-count")[:10]    ):        if event["data__browser"] != "" and event["data__browser"] is not None:            total_events_by_browser.append(                {"label": event["data__browser"], "count": event["count"]}            )    context["total_events_by_browser"] = total_events_by_browser[:7]    #    # Total events by platform graph    #    total_events_by_platform = []    for event in (        events_filtered.filter(event="session_start")        .exclude(data__platform__isnull=True)        .values("data__platform")        .annotate(count=models.Count("data__platform"))        .order_by("-count")[:10]    ):        if event["data__platform"] != "" and event["data__platform"] is not None:            total_events_by_platform.append(                {"label": event["data__platform"], "count": event["count"]}            )    context["total_events_by_platform"] = total_events_by_platform[:7]    #    # Total events by page url list    #    total_events_by_page_url = []    for page_view in (        events_filtered.filter(data__url__isnull=False)        .values("data__url")        .annotate(count=models.Count("data__url"))        .order_by("-count")[:10]    ):        total_events_by_page_url.append(            {"label": page_view["data__url"], "count": page_view["count"]}        )    context["total_events_by_page_url"] = total_events_by_page_url    #    # Total events by page view    #    total_page_views_by_page_url = []    for page_view in (        events_filtered.filter(event="page_view")        .exclude(data__url__isnull=True)        .values("data__url")        .annotate(count=models.Count("data__url"))        .order_by("-count")[:10]    ):        total_page_views_by_page_url.append(            {"label": page_view["data__url"], "count": page_view["count"]}        )    context["total_page_views_by_page_url"] = total_page_views_by_page_url    #    # Total events by custom event list    #    total_events_by_custom_event = []    for custom_event in (        events_filtered.exclude(            event__in=["session_start", "page_view", "page_leave", "click", "scroll"]        )        .values("event")        .annotate(count=models.Count("event"))        .order_by("-count")[:10]    ):        total_events_by_custom_event.append(            {"label": custom_event["event"], "count": custom_event["count"]}        )    context["total_events_by_custom_event"] = total_events_by_custom_event    #    # Total session starts by referrer list    #    total_session_starts_by_referrer = []    for referrer in (        events_filtered.filter(event="session_start")        .exclude(data__referrer__isnull=True, data__referrer="")        .values("data__referrer")        .annotate(count=models.Count("data__referrer"))        .order_by("-count")[:10]    ):        if referrer["data__referrer"] != "" and referrer["data__referrer"] is not None:            total_session_starts_by_referrer.append(                {"label": referrer["data__referrer"], "count": referrer["count"]}            )    context["total_session_starts_by_referrer"] = total_session_starts_by_referrer    #    # Total page views by utm_medium list    #    total_page_views_by_utm_medium = []    for utm_medium in (        events_filtered.filter(event="page_view")        .exclude(data__utm_medium__isnull=True, data__utm_medium="")        .values("data__utm_medium")        .annotate(count=models.Count("data__utm_medium"))        .order_by("-count")[:10]    ):        if (            utm_medium["data__utm_medium"] != ""            and utm_medium["data__utm_medium"] is not None        ):            total_page_views_by_utm_medium.append(                {"label": utm_medium["data__utm_medium"], "count": utm_medium["count"]}            )    context["total_page_views_by_utm_medium"] = total_page_views_by_utm_medium    #    # Total page views by utm_source list    #    total_page_views_by_utm_source = []    for utm_source in (        events_filtered.filter(event="page_view")        .exclude(data__utm_source__isnull=True, data__utm_source="")        .values("data__utm_source")        .annotate(count=models.Count("data__utm_source"))        .order_by("-count")[:10]    ):        if (            utm_source["data__utm_source"] != ""            and utm_source["data__utm_source"] is not None        ):            total_page_views_by_utm_source.append(                {"label": utm_source["data__utm_source"], "count": utm_source["count"]}            )    context["total_page_views_by_utm_source"] = total_page_views_by_utm_source    #    # Total page views by utm_campaign list    #    total_page_views_by_utm_campaign = []    for utm_campaign in (        events_filtered.filter(event="page_view")        .exclude(data__utm_campaign__isnull=True, data__utm_campaign="")        .values("data__utm_campaign")        .annotate(count=models.Count("data__utm_campaign"))        .order_by("-count")[:10]    ):        if (            utm_campaign["data__utm_campaign"] != ""            and utm_campaign["data__utm_campaign"] is not None        ):            total_page_views_by_utm_campaign.append(                {                    "label": utm_campaign["data__utm_campaign"],                    "count": utm_campaign["count"],                }            )    context["total_page_views_by_utm_campaign"] = total_page_views_by_utm_campaign    #    # Total session starts by region list    #    total_session_starts_by_region = []    for region in (        events_filtered.filter(event="session_start")        .exclude(data__region__isnull=True)        .values("data__region")        .annotate(count=models.Count("data__region"))        .order_by("-count")    ):        total_session_starts_by_region.append(            {"label": region["data__region"], "count": region["count"]}        )    context["total_session_starts_by_region"] = total_session_starts_by_region[:10]    #    # Total session starts by region map    #    # For datamaps we want to convert each state label to a two letter    # abbreviation and format the data in a dict such as:    # {'AL': {'count': 23}, 'AK': {'count': 14}, '...'}    us_states = {        "Alabama": "AL",        "Alaska": "AK",        "Arizona": "AZ",        "Arkansas": "AR",        "California": "CA",        "Colorado": "CO",        "Connecticut": "CT",        "Delaware": "DE",        "Florida": "FL",        "Georgia": "GA",        "Hawaii": "HI",        "Idaho": "ID",        "Illinois": "IL",        "Indiana": "IN",        "Iowa": "IA",        "Kansas": "KS",        "Kentucky": "KY",        "Louisiana": "LA",        "Maine": "ME",        "Maryland": "MD",        "Massachusetts": "MA",        "Michigan": "MI",        "Minnesota": "MN",        "Mississippi": "MS",        "Missouri": "MO",        "Montana": "MT",        "Nebraska": "NE",        "Nevada": "NV",        "New Hampshire": "NH",        "New Jersey": "NJ",        "New Mexico": "NM",        "New York": "NY",        "North Carolina": "NC",        "North Dakota": "ND",        "Ohio": "OH",        "Oklahoma": "OK",        "Oregon": "OR",        "Pennsylvania": "PA",        "Rhode Island": "RI",        "South Carolina": "SC",        "South Dakota": "SD",        "Tennessee": "TN",        "Texas": "TX",        "Utah": "UT",        "Vermont": "VT",        "Virginia": "VA",        "Washington": "WA",        "West Virginia": "WV",        "Wisconsin": "WI",        "Wyoming": "WY",        "District of Columbia": "DC",        "American Samoa": "AS",        "Guam": "GU",        "Northern Mariana Islands": "MP",        "Puerto Rico": "PR",        "United States Minor Outlying Islands": "UM",        "U.S. Virgin Islands": "VI",        dashboard = cache.get(cache_key)        if dashboard is None:            dashboard = _dashboard_context(property_obj, date_start_obj, date_end_obj, date_range, filter_url)            cache.set(cache_key, dashboard, DASHBOARD_CACHE_TTL)    context = {        "property": property_obj,        "title": property_obj.name,        "description": "Analytics for " + property_obj.name,        "BASE_URL": settings.BASE_URL,        "date_start": date_start,        "date_end": date_end,        "date_range": date_range,        "filter_url": filter_url,        "total_live_users": q.total_live_users(property_obj),        **dashboard,    }    total_session_starts_by_region_chart_data = {}    for region in total_session_starts_by_region:        if region["label"] in us_states:            total_session_starts_by_region_chart_data[us_states[region["label"]]] = {                "numberOfThings": region["count"]            }        elif region["label"] in us_states.values():            total_session_starts_by_region_chart_data[region["label"]] = {                "numberOfThings": region["count"]            }    context[        "total_session_starts_by_region_chart_data"    ] = total_session_starts_by_region_chart_data    # Report formats. `?report` (or `?report=pdf`) returns a printed PDF    # rendered from the dashboard template; `?report=md` returns a plain-text    # Markdown report suited for piping into an LLM.    if "report" in request.GET:        fmt = request.GET.get("report") or "pdf"        if fmt == "md":