19.3 KB
raw
use axum::{
extract::{Path as AxumPath, Query, State},
http::{header, HeaderMap, StatusCode},
response::{IntoResponse, Json, Redirect, Response},
routing::{get, post},
Router,
};
use serde::Deserialize;
use serde_json::{json, Value};
use std::collections::HashMap;
use tower_cookies::Cookies;
use uuid::Uuid;
use crate::models::{
self, count_checks, count_status_codes, count_uptime, ms_to_iso_opt, recent_checks,
PropertyContext, PropertyRow,
};
use crate::render::{render, render_to_string};
use crate::routes::auth::is_authenticated;
use crate::AppState;
pub fn router() -> Router<AppState> {
Router::new()
.route("/properties/{id}/status", get(property_status))
.route("/properties/{id}/recrawl", post(property_recrawl))
.route(
"/properties/{id}/rerun-lighthouse",
post(property_rerun_lighthouse),
)
// UUID catch-all is the property dashboard. Keep the merge order in
// app::router so the named routes above win the match.
.route("/{property_id}", get(property))
}
fn forbidden_json() -> Response {
(StatusCode::FORBIDDEN, Json(json!({"error": "forbidden"}))).into_response()
}
fn not_found_json() -> Response {
(StatusCode::NOT_FOUND, Json(json!({"error": "not_found"}))).into_response()
}
#[derive(Debug, Deserialize)]
pub struct PropertyQuery {
pub report: Option<String>,
}
pub async fn property(
State(state): State<AppState>,
cookies: Cookies,
AxumPath(property_id): AxumPath<Uuid>,
Query(q): Query<PropertyQuery>,
) -> Response {
let row = match models::get_property(&state.pool, property_id).await {
Ok(Some(r)) => r,
_ => return Redirect::to("/properties").into_response(),
};
let authed = is_authenticated(&cookies, &state);
let public = row.is_public != 0;
if !public && !authed {
return Redirect::to("/properties").into_response();
}
let ctx = match build_property_context(&state, &row).await {
Ok(c) => c,
Err(e) => {
tracing::error!("property context: {e:#}");
return (StatusCode::INTERNAL_SERVER_ERROR, "context error").into_response();
}
};
// Per-page graphs (response times, status codes, uptime).
let recent = recent_checks(&state.pool, property_id, 31).await.unwrap_or_default();
let status_response_times: Vec<Value> = recent
.iter()
.rev()
.map(|c| {
json!({
"label": chrono::DateTime::<chrono::Utc>::from_timestamp_millis(c.created_at)
.map(|d| d.to_rfc3339())
.unwrap_or_default(),
"total": c.response_ms,
"dns": c.dns_ms,
"tcp": c.tcp_ms,
"tls": c.tls_ms,
"ttfb": c.ttfb_ms,
})
})
.collect();
let codes = count_status_codes(&state.pool, property_id).await.unwrap_or_default();
let status_codes_graph: Vec<Value> = codes
.iter()
.map(|(code, count)| json!({"label": code, "count": count}))
.collect();
let (up, down) = count_uptime(&state.pool, property_id).await.unwrap_or((0, 0));
let total = up + down;
let pct = |n: i64| -> f64 {
if total == 0 {
0.0
} else {
(n as f64 / total as f64 * 10000.0).round() / 100.0
}
};
let uptime_graph: Vec<Value> = vec![
json!({"label": "Uptime", "count": pct(up)}),
json!({"label": "Downtime", "count": pct(down)}),
];
let title = ctx.name.clone();
let description = format!("Status for {}", ctx.name);
let property_value = serde_json::to_value(&ctx).unwrap_or(Value::Null);
let insights_groups = group_insights_by_type(&ctx.crawler_insights);
let path = format!("/{property_id}");
// Report formats.
if let Some(fmt) = q.report.as_deref() {
let fmt = if fmt.is_empty() { "pdf" } else { fmt };
if fmt == "md" {
let extra = minijinja::context! {
property => &property_value,
title => &title,
description => &description,
};
return match render_to_string(
&state,
"properties/property_report.md",
&path,
authed,
extra,
) {
Ok(body) => {
let mut h = HeaderMap::new();
h.insert(
header::CONTENT_TYPE,
"text/markdown; charset=utf-8".parse().unwrap(),
);
h.insert(
header::CONTENT_DISPOSITION,
format!("inline; filename=\"{}.md\"", ctx.name).parse().unwrap(),
);
(StatusCode::OK, h, body).into_response()
}
Err(resp) => resp,
};
}
if fmt == "pdf" {
let extra = minijinja::context! {
property => &property_value,
insights_groups => &insights_groups,
title => &title,
description => &description,
base_url => &state.config.base_url,
generated_at => chrono::Local::now().format("%Y-%m-%d %H:%M %Z").to_string(),
};
let typst_source = match render_to_string(
&state,
"properties/property_report.typ",
&path,
authed,
extra,
) {
Ok(s) => s,
Err(resp) => return resp,
};
let renderer = state.pdf_renderer.clone();
let result =
tokio::task::spawn_blocking(move || renderer.render(typst_source)).await;
return match result {
Ok(Ok(bytes)) => {
let mut hh = HeaderMap::new();
hh.insert(header::CONTENT_TYPE, "application/pdf".parse().unwrap());
hh.insert(
header::CONTENT_DISPOSITION,
format!("inline; filename=\"{}.pdf\"", ctx.name).parse().unwrap(),
);
(StatusCode::OK, hh, bytes).into_response()
}
Ok(Err(e)) => {
tracing::error!("pdf render: {e:#}");
(StatusCode::SERVICE_UNAVAILABLE, "pdf unavailable").into_response()
}
Err(e) => {
tracing::error!("pdf join: {e}");
(StatusCode::INTERNAL_SERVER_ERROR, "pdf join error").into_response()
}
};
}
}
let extra = minijinja::context! {
page => minijinja::context! { title => &title, description => &description },
property => &property_value,
status_response_times_graph => &status_response_times,
status_codes_graph => &status_codes_graph,
uptime_graph => &uptime_graph,
insights_groups => &insights_groups,
title => &title,
description => &description,
};
render(&state, "properties/property.html", &path, authed, extra)
}
pub async fn build_property_context(
state: &AppState,
row: &PropertyRow,
) -> anyhow::Result<PropertyContext> {
let id = row.uuid();
let recent = recent_checks(&state.pool, id, 100).await?;
let total = count_checks(&state.pool, id).await?;
let current_status = recent.first().map(|c| c.status_code).unwrap_or(200);
let avg_response_time = if recent.is_empty() {
0
} else {
let n = recent.iter().take(31).count() as i64;
let sum: i64 = recent.iter().take(31).map(|c| c.response_ms).sum();
if n == 0 {
0
} else {
sum / n
}
};
let recent_uptime_pct = if recent.is_empty() {
None
} else {
let up = recent.iter().filter(|c| c.status_code == 200).count() as f64;
Some(((up / recent.len() as f64) * 1000.0).round() / 10.0)
};
let mut tick: Vec<&'static str> = recent
.iter()
.rev()
.map(|c| if c.status_code == 200 { "up" } else { "down" })
.collect();
tick.truncate(30);
let latest_headers: HashMap<String, String> = recent
.first()
.and_then(|c| serde_json::from_str::<HashMap<String, String>>(&c.headers).ok())
.unwrap_or_default();
let lower: HashMap<String, String> = latest_headers
.into_iter()
.map(|(k, v)| (k.to_lowercase(), v.to_lowercase()))
.collect();
let is_https = row.url.starts_with("https://");
let invalid_cert = current_status == 526;
let has_mime_type = lower.contains_key("content-type");
let has_content_sniffing_protection = lower
.get("x-content-type-options")
.map(|v| v == "nosniff")
.unwrap_or(false);
let has_clickjack_protection = lower
.get("x-frame-options")
.map(|v| matches!(v.as_str(), "deny" | "sameorigin" | "allow-from"))
.unwrap_or(false);
let hides_server_version = !lower.contains_key("server")
&& !lower.contains_key("x-server")
&& !lower.contains_key("powered-by")
&& !lower.contains_key("x-powered-by");
let hsts = lower
.get("strict-transport-security")
.cloned()
.unwrap_or_default();
let has_hsts = {
if hsts.is_empty() {
false
} else if let Some(re) = regex::Regex::new(r"max-age=(\d+)").ok() {
re.captures(&hsts)
.and_then(|c| c.get(1))
.and_then(|m| m.as_str().parse::<i64>().ok())
.map(|m| m >= 31_536_000)
.unwrap_or(false)
} else {
false
}
};
let has_hsts_preload = hsts.to_lowercase().contains("preload");
let has_security_issue = !is_https
|| !has_mime_type
|| !has_content_sniffing_protection
|| !has_clickjack_protection
|| !hides_server_version
|| !has_hsts
|| !has_hsts_preload;
let lighthouse_scores: Value = row
.lighthouse_scores
.as_ref()
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(Value::Null);
let lighthouse_details: Value = row
.lighthouse_details
.as_ref()
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(Value::Null);
let crawler_insights: Value = row
.crawler_insights
.as_ref()
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(Value::Array(Vec::new()));
let avg_lighthouse_score: Option<i64> = lighthouse_scores.as_object().and_then(|m| {
let scores: Vec<i64> = m.values().filter_map(|v| v.as_i64()).collect();
if scores.is_empty() {
None
} else {
Some((scores.iter().sum::<i64>() as f64 / scores.len() as f64).round() as i64)
}
});
Ok(PropertyContext {
id: row.uuid().to_string(),
url: row.url.clone(),
name: row.name(),
is_public: row.is_public != 0,
is_protected: row.is_protected != 0,
current_status,
avg_response_time,
recent_uptime_pct,
recent_tick_stream: tick,
total_checks: total,
crawl_state: row.crawl_state.clone(),
crawler_insights,
last_crawl_success_at: ms_to_iso_opt(row.last_crawl_success_at),
last_crawl_error: row.last_crawl_error.clone(),
last_crawl_duration_ms: row.last_crawl_duration_ms,
last_crawl_pages_count: row.last_crawl_pages_count,
next_run_at_crawler: ms_to_iso_opt(row.next_run_at_crawler),
crawl_started_at: ms_to_iso_opt(row.crawl_started_at),
lighthouse_state: row.lighthouse_state.clone(),
lighthouse_scores,
lighthouse_details,
last_lighthouse_success_at: ms_to_iso_opt(row.last_lighthouse_success_at),
last_lighthouse_error: row.last_lighthouse_error.clone(),
last_lighthouse_duration_ms: row.last_lighthouse_duration_ms,
next_lighthouse_run_at: ms_to_iso_opt(row.next_lighthouse_run_at),
lighthouse_started_at: ms_to_iso_opt(row.lighthouse_started_at),
avg_lighthouse_score,
alert_state: row.alert_state.clone(),
created_at: models::ms_to_iso(row.created_at),
updated_at: models::ms_to_iso(row.updated_at),
is_https,
invalid_cert,
has_mime_type,
has_content_sniffing_protection,
has_clickjack_protection,
hides_server_version,
has_hsts,
has_hsts_preload,
has_security_issue,
})
}
fn group_insights_by_type(insights: &Value) -> Vec<Value> {
use std::collections::BTreeMap;
let arr = match insights.as_array() {
Some(a) => a,
None => return Vec::new(),
};
let mut buckets: BTreeMap<String, Vec<Value>> = BTreeMap::new();
for item in arr {
let t = item
.get("type")
.and_then(|v| v.as_str())
.unwrap_or("other")
.to_string();
buckets.entry(t).or_default().push(item.clone());
}
// Sort each bucket so errors come before warnings before info, matching the
// old Django dictsort:"severity".
let sev_rank = |s: &str| -> u8 {
match s {
"error" => 0,
"warning" => 1,
_ => 2,
}
};
for items in buckets.values_mut() {
items.sort_by_key(|i| sev_rank(i.get("severity").and_then(|v| v.as_str()).unwrap_or("info")));
}
buckets
.into_iter()
.map(|(name, items)| json!({"type": name, "items": items}))
.collect()
}
fn crawl_progress(prop: &PropertyRow) -> f64 {
let pages = prop.last_crawl_pages_count.unwrap_or(0);
if pages <= 0 {
return 0.05; // show *some* movement once we start
}
let cap = crate::crawler::PAGE_CAP as f64;
((pages as f64) / cap).min(0.9)
}
fn serialize_status(prop: &PropertyRow) -> Value {
let now = chrono::Utc::now().timestamp_millis();
let insights: Value = prop
.crawler_insights
.as_ref()
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(Value::Array(Vec::new()));
let mut sev = serde_json::Map::new();
sev.insert("error".into(), 0.into());
sev.insert("warning".into(), 0.into());
sev.insert("info".into(), 0.into());
let mut total = 0i64;
if let Some(arr) = insights.as_array() {
for i in arr {
total += 1;
let s = i.get("severity").and_then(|v| v.as_str()).unwrap_or("info");
if let Some(n) = sev.get_mut(s).and_then(|v| v.as_i64()) {
sev.insert(s.into(), Value::from(n + 1));
}
}
}
let crawl_next = prop.next_run_at_crawler;
let lh_next = prop.next_lighthouse_run_at;
json!({
"crawler": {
"state": prop.crawl_state,
"started_at": ms_to_iso_opt(prop.crawl_started_at),
"last_attempt_at": ms_to_iso_opt(prop.last_run_at_crawler),
"last_success_at": ms_to_iso_opt(prop.last_crawl_success_at),
"last_error": prop.last_crawl_error,
"last_duration_ms": prop.last_crawl_duration_ms,
"pages_count": prop.last_crawl_pages_count,
"next_run_at": ms_to_iso_opt(crawl_next),
"is_overdue": crawl_next.map(|n| n <= now).unwrap_or(false),
"insights_total": total,
"insights_by_severity": Value::Object(sev),
"progress": if prop.crawl_state == "running" {
Value::from(crawl_progress(prop))
} else {
Value::Null
},
},
"lighthouse": {
"state": prop.lighthouse_state,
"started_at": ms_to_iso_opt(prop.lighthouse_started_at),
"last_attempt_at": ms_to_iso_opt(prop.last_lighthouse_run_at),
"last_success_at": ms_to_iso_opt(prop.last_lighthouse_success_at),
"last_error": prop.last_lighthouse_error,
"last_duration_ms": prop.last_lighthouse_duration_ms,
"next_run_at": ms_to_iso_opt(lh_next),
"is_overdue": lh_next.map(|n| n <= now).unwrap_or(false),
"scores": prop.lighthouse_scores
.as_ref()
.and_then(|s| serde_json::from_str::<Value>(s).ok())
.unwrap_or(Value::Null),
},
"server_time": chrono::Utc::now().to_rfc3339(),
})
}
pub async fn property_status(
State(state): State<AppState>,
cookies: Cookies,
AxumPath(id): AxumPath<Uuid>,
) -> Response {
let row = match models::get_property(&state.pool, id).await {
Ok(Some(r)) => r,
_ => return not_found_json(),
};
let authed = is_authenticated(&cookies, &state);
if row.is_public == 0 && !authed {
return forbidden_json();
}
Json(serialize_status(&row)).into_response()
}
pub async fn property_recrawl(
State(state): State<AppState>,
cookies: Cookies,
AxumPath(id): AxumPath<Uuid>,
) -> Response {
if !is_authenticated(&cookies, &state) {
return forbidden_json();
}
let row = match models::get_property(&state.pool, id).await {
Ok(Some(r)) => r,
_ => return not_found_json(),
};
if matches!(row.crawl_state.as_str(), "queued" | "running") {
return Json(json!({
"ok": false,
"reason": "already_running",
"crawler": serialize_status(&row).get("crawler"),
"lighthouse": serialize_status(&row).get("lighthouse"),
"server_time": chrono::Utc::now().to_rfc3339(),
}))
.into_response();
}
let now = chrono::Utc::now().timestamp_millis();
let _ = sqlx::query(
"UPDATE properties SET next_run_at_crawler = ?, last_crawl_error = NULL, updated_at = ? WHERE id = ?",
)
.bind(now)
.bind(now)
.bind(row.id.clone())
.execute(&state.pool)
.await;
let updated = models::get_property(&state.pool, id)
.await
.ok()
.flatten()
.unwrap_or(row);
let mut payload = serialize_status(&updated);
if let Some(obj) = payload.as_object_mut() {
obj.insert("ok".into(), Value::Bool(true));
}
Json(payload).into_response()
}
pub async fn property_rerun_lighthouse(
State(state): State<AppState>,
cookies: Cookies,
AxumPath(id): AxumPath<Uuid>,
) -> Response {
if !is_authenticated(&cookies, &state) {
return forbidden_json();
}
let row = match models::get_property(&state.pool, id).await {
Ok(Some(r)) => r,
_ => return not_found_json(),
};
if matches!(row.lighthouse_state.as_str(), "queued" | "running") {
return Json(json!({
"ok": false,
"reason": "already_running",
"crawler": serialize_status(&row).get("crawler"),
"lighthouse": serialize_status(&row).get("lighthouse"),
"server_time": chrono::Utc::now().to_rfc3339(),
}))
.into_response();
}
let now = chrono::Utc::now().timestamp_millis();
let _ = sqlx::query(
"UPDATE properties SET next_lighthouse_run_at = ?, last_lighthouse_error = NULL, updated_at = ? WHERE id = ?",
)
.bind(now)
.bind(now)
.bind(row.id.clone())
.execute(&state.pool)
.await;
let updated = models::get_property(&state.pool, id)
.await
.ok()
.flatten()
.unwrap_or(row);
let mut payload = serialize_status(&updated);
if let Some(obj) = payload.as_object_mut() {
obj.insert("ok".into(), Value::Bool(true));
}
Json(payload).into_response()
}