From 2bcb62f67572ce404197fb74c9b72ec592e10e6b Mon Sep 17 00:00:00 2001 From: X99 Date: Thu, 1 Oct 2026 20:09:41 +0200 Subject: [PATCH] Signal and memory history in the details page Otter keeps a sample of each device's signal and free memory every 5 minutes for 7 days, from its check-ins; a restart adds one at once with its reset reason. The details page charts them over 24 hours or 7 days (half-hour means over a week), one chart per measure, restarts marked (red after a crash), the line broken while the device was away, and the nearest sample shown on hover. Fixes #98 --- README.md | 7 + server/otter/api_device.py | 5 +- server/otter/api_ui.py | 21 ++- server/otter/history.py | 50 +++++++ .../versions/0021_device_samples.py | 39 +++++ server/otter/models.py | 18 ++- server/otter/static/app.js | 136 ++++++++++++++++++ server/otter/static/index.html | 15 ++ server/otter/static/style.css | 23 +++ server/tests/test_history.py | 67 +++++++++ 10 files changed, 377 insertions(+), 4 deletions(-) create mode 100644 server/otter/history.py create mode 100644 server/otter/migrations/versions/0021_device_samples.py create mode 100644 server/tests/test_history.py diff --git a/README.md b/README.md index f1144e3..fe6c648 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,13 @@ Espressif's `esp_delta_ota`): a few tens of KB instead of a megabyte. The device really runs the patch's base first, and a retry downloads the whole image. Nothing to configure: the smallest transfer is picked each time. Other agents keep downloading the plain image. +## History + +A device's details page charts its Wi-Fi signal and free memory over the last 24 hours or 7 days, +with its restarts marked (in red after a crash): a memory leak, or a board that drops off at +night, shows at a glance. Otter keeps a sample per device every 5 minutes for 7 days, from the +check-ins (a few MB for a hundred devices). + ## Live logs **Live logs** in a device's details page shows its log output as it runs, without a USB cable: diff --git a/server/otter/api_device.py b/server/otter/api_device.py index 7909906..1f77eaa 100644 --- a/server/otter/api_device.py +++ b/server/otter/api_device.py @@ -12,7 +12,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from . import commands, config, crashes, devconfig +from . import commands, config, crashes, devconfig, history from .auth import token_hash from .db import SessionLocal, get_session, utcnow from .device_auth import DeviceAuth, authenticate, check_access, new_token @@ -151,6 +151,9 @@ def record_checkin( if is_new: notifier.emit("device_new", device_id=device.id) raise HTTPException(403, "device awaiting approval in Otter") + if device.id is None: + session.flush() + history.record(session, device, restart=(body.reset_reason or "restart") if rebooted else None) # Until the device uses a token, each fleet-key check-in gets a fresh one. token = None if auth.fleet and device.token_used_at is None: diff --git a/server/otter/api_ui.py b/server/otter/api_ui.py index e251d40..6a89c24 100644 --- a/server/otter/api_ui.py +++ b/server/otter/api_ui.py @@ -1,15 +1,16 @@ """Endpoints used by the web UI.""" import asyncio +from datetime import UTC import re -from fastapi import APIRouter, Depends, Form, HTTPException, Request, UploadFile +from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse, StreamingResponse from sqlalchemy import delete, func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, selectinload -from . import commands, config, crashes, devconfig, signing +from . import commands, config, crashes, devconfig, history, signing from .api_device import publish_device from .api_rollouts import reconcile_channels from .auth import require_user @@ -237,9 +238,25 @@ def delete_device(device_id: int, session: Session = Depends(get_session)): delete_unused_tags(session) session.commit() device_logs.forget(device_id) + history.forget(device_id) broadcaster.publish("device_deleted", {"id": device_id}) +# --- History (#98, see history.py) --------------------------------------------- + + +@router.get("/devices/{device_id}/history") +def device_history(device_id: int, hours: int = Query(24, ge=1, le=168), session: Session = Depends(get_session)): + """Signal and free memory over the last hours, and the restarts in that time.""" + session.get(Device, device_id) or _404("device") + rows = history.samples(session, device_id, hours) + stamp = lambda at: at.replace(tzinfo=UTC).isoformat() # noqa: E731 + return { + "samples": [{"t": stamp(s.at), "rssi": s.rssi, "free_heap": s.free_heap} for s in rows], + "restarts": [{"t": stamp(s.at), "reason": s.restart} for s in rows if s.restart], + } + + # --- Live logs (#96, see logs.py) --------------------------------------------- diff --git a/server/otter/history.py b/server/otter/history.py new file mode 100644 index 0000000..5753403 --- /dev/null +++ b/server/otter/history.py @@ -0,0 +1,50 @@ +"""Signal and free memory history (#98): a sample per device every SAMPLE_EVERY, kept KEEP. + +A restart forces a sample at once, carrying its reset reason, so the charts can mark it. +""" + +from datetime import timedelta + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from .db import utcnow +from .models import Device, DeviceSample + +SAMPLE_EVERY = timedelta(minutes=5) +KEEP = timedelta(days=7) +PRUNE_EVERY = timedelta(hours=1) + +_last_sample: dict[int, object] = {} # device id -> time of its last sample, this process +_pruned_at = None + + +def record(session: Session, device: Device, restart: str | None = None) -> None: + """Called at each check-in; writes a sample when one is due.""" + global _pruned_at + now = utcnow() + last = _last_sample.get(device.id) + if restart is None and (last is not None and now - last < SAMPLE_EVERY): + return + if restart is None and device.rssi is None and device.free_heap is None: + return + session.add(DeviceSample(device_id=device.id, at=now, rssi=device.rssi, free_heap=device.free_heap, restart=restart)) + _last_sample[device.id] = now + if _pruned_at is None or now - _pruned_at > PRUNE_EVERY: + session.execute(delete(DeviceSample).where(DeviceSample.at < now - KEEP)) + _pruned_at = now + + +def samples(session: Session, device_id: int, hours: int) -> list[DeviceSample]: + since = utcnow() - timedelta(hours=hours) + return list( + session.scalars( + select(DeviceSample) + .where(DeviceSample.device_id == device_id, DeviceSample.at >= since) + .order_by(DeviceSample.at) + ) + ) + + +def forget(device_id: int) -> None: + _last_sample.pop(device_id, None) diff --git a/server/otter/migrations/versions/0021_device_samples.py b/server/otter/migrations/versions/0021_device_samples.py new file mode 100644 index 0000000..5032451 --- /dev/null +++ b/server/otter/migrations/versions/0021_device_samples.py @@ -0,0 +1,39 @@ +"""device samples + +Revision ID: 0021 +Revises: 0020 +Create Date: 2026-10-01 22:00:00.000000 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + + +revision: str = '0021' +down_revision: str | None = '0020' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + 'device_samples', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('device_id', sa.Integer(), nullable=False), + sa.Column('at', sa.DateTime(), nullable=False), + sa.Column('rssi', sa.Integer(), nullable=True), + sa.Column('free_heap', sa.Integer(), nullable=True), + sa.Column('restart', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['device_id'], ['devices.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_device_samples_at', 'device_samples', ['at']) + op.create_index('ix_device_samples_device_at', 'device_samples', ['device_id', 'at']) + + +def downgrade() -> None: + op.drop_index('ix_device_samples_device_at', table_name='device_samples') + op.drop_index('ix_device_samples_at', table_name='device_samples') + op.drop_table('device_samples') diff --git a/server/otter/models.py b/server/otter/models.py index f084c57..59dad75 100644 --- a/server/otter/models.py +++ b/server/otter/models.py @@ -1,7 +1,7 @@ import json from datetime import datetime -from sqlalchemy import Column, ForeignKey, String, Table, UniqueConstraint +from sqlalchemy import Column, ForeignKey, Index, String, Table, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from .db import Base, utcnow @@ -253,6 +253,22 @@ class ConfigValue(Base): value: Mapped[str] # JSON: a string, number or boolean +class DeviceSample(Base): + """A device's signal and free memory at one check-in (#98): one every few minutes, kept a + week (see history.py). restart is the reset reason when the device had just restarted.""" + + __tablename__ = "device_samples" + + id: Mapped[int] = mapped_column(primary_key=True) + device_id: Mapped[int] = mapped_column(ForeignKey("devices.id", ondelete="CASCADE")) + at: Mapped[datetime] = mapped_column(default=utcnow, index=True) + rssi: Mapped[int | None] + free_heap: Mapped[int | None] + restart: Mapped[str | None] + + __table_args__ = (Index("ix_device_samples_device_at", "device_id", "at"),) + + class Crash(Base): """A crash report (#24): the core dump summary a device sent after restarting from a crash.""" diff --git a/server/otter/static/app.js b/server/otter/static/app.js index c3b5349..7c98c9a 100644 --- a/server/otter/static/app.js +++ b/server/otter/static/app.js @@ -525,6 +525,9 @@ function openPanel(id) { panel.focus(); // not its first button, which would show a focus ring loadCommands(id); loadLogs(id); + historyData = null; + panel.querySelectorAll(".history-charts .plot").forEach((p) => { p.innerHTML = ""; }); + loadDeviceHistory(id); loadConfig(id, "device"); loadCrashes(id); } @@ -1417,3 +1420,136 @@ $(".logs-toggle", panel).addEventListener("click", () => { watchLogs(); logsRenew = setInterval(watchLogs, LOGS_RENEW_MS); }); + +// --- History charts (#98) ------------------------------------------------------ +// A sample every 5 minutes: signal and free memory, one chart each (never two scales on one), +// restarts marked. Hovering shows the nearest sample. + +let historyHours = 24; +let historyData = null; +const SAMPLE_GAP_MS = 16 * 60 * 1000; // more than 3 missed samples: the device was away +const CHARTS = { + rssi: { value: (s) => s.rssi, format: (v) => `${v} dBm` }, + free_heap: { value: (s) => (s.free_heap == null ? null : s.free_heap / 1024), format: (v) => `${Math.round(v)} KB` }, +}; + +function niceTicks(min, max, count = 3) { + const span = max - min || 1; + const step = [1, 2, 5, 10, 20, 25, 50, 100, 200, 250, 500, 1000].find((s) => span / s <= count) || span / count; + const ticks = []; + for (let v = Math.ceil(min / step) * step; v <= max; v += step) ticks.push(v); + return ticks; +} + +// Over a week, 5-minute samples are noise: their mean per bucket (a gap still breaks the line). +function averaged(points, bucketMs) { + const buckets = new Map(); + for (const p of points) { + const key = Math.floor(p.t / bucketMs); + const b = buckets.get(key) || buckets.set(key, { t: 0, v: 0, n: 0 }).get(key); + b.t += p.t; b.v += p.v; b.n += 1; + } + return [...buckets.values()].map((b) => ({ t: b.t / b.n, v: b.v / b.n })); +} + +function drawChart(figure, samples, restarts, hours) { + const plot = $(".plot", figure); + const { value, format } = CHARTS[figure.dataset.key]; + let points = samples.map((s) => ({ t: Date.parse(s.t), v: value(s) })).filter((p) => p.v != null); + if (hours > 24) points = averaged(points, 30 * 60 * 1000); + if (points.length < 2) { + plot.innerHTML = `

${points.length ? "One sample so far" : "No samples yet"}: one comes every 5 minutes.

`; + return; + } + const W = Math.max(240, plot.clientWidth), H = 150, L = 40, R = 8, T = 8, B = 22; + const end = Date.now(), start = end - hours * 3600 * 1000; + let lo = Math.min(...points.map((p) => p.v)), hi = Math.max(...points.map((p) => p.v)); + const pad = (hi - lo) * 0.15 || Math.abs(hi) * 0.05 || 1; + lo -= pad; hi += pad; + const x = (t) => L + ((t - start) / (end - start)) * (W - L - R); + const y = (v) => T + (1 - (v - lo) / (hi - lo)) * (H - T - B); + + // Grid and axes: recessive. + const yTicks = niceTicks(lo, hi).map((v) => + `${Math.round(v)}`); + const stepH = hours <= 24 ? 6 : 24; + const xTicks = []; + const first = new Date(start); first.setMinutes(0, 0, 0); + for (let t = first.getTime() + 3600e3; t < end; t += 3600e3) { + const d = new Date(t); + if (d.getHours() % stepH) continue; + const text = stepH === 24 ? d.toLocaleDateString([], { weekday: "short" }) : `${String(d.getHours()).padStart(2, "0")}:00`; + xTicks.push(`${text}`); + } + // The line, broken where the device was away. + let path = ""; + points.forEach((p, i) => { + const gap = i === 0 || p.t - points[i - 1].t > (hours > 24 ? 2.5 * 30 * 60 * 1000 : SAMPLE_GAP_MS); + path += `${gap ? "M" : "L"}${x(p.t).toFixed(1)},${y(p.v).toFixed(1)}`; + }); + const marks = restarts.map((r) => { + const crash = CRASH_RESETS.has(r.reason); + const rx = x(Date.parse(r.t)); + return `Restarted: ${esc(r.reason.replaceAll("_", " "))}`; + }); + plot.innerHTML = ` + p.v)))} to ${format(Math.max(...points.map((p) => p.v)))}, last ${format(points.at(-1).v)}"> + ${yTicks.join("")}${xTicks.join("")}${marks.join("")} + + + + + `; + const svg = $("svg", plot), hover = $(".hover", svg), tip = $(".chart-tip", plot); + $(".hit", svg).addEventListener("mousemove", (e) => { + const box = svg.getBoundingClientRect(); + const t = start + ((e.clientX - box.left) * (W / box.width) - L) / (W - L - R) * (end - start); + const p = points.reduce((a, b) => (Math.abs(b.t - t) < Math.abs(a.t - t) ? b : a)); + const px = x(p.t), py = y(p.v); + hover.setAttribute("visibility", "visible"); + $(".crosshair", hover).setAttribute("x1", px); $(".crosshair", hover).setAttribute("x2", px); + $("circle", hover).setAttribute("cx", px); $("circle", hover).setAttribute("cy", py); + const when = new Date(p.t).toLocaleString([], { weekday: hours > 24 ? "short" : undefined, hour: "2-digit", minute: "2-digit" }); + tip.innerHTML = `${format(p.v)} ${when}`; + tip.hidden = false; + tip.style.left = `${Math.min(px * (box.width / W), box.width - tip.offsetWidth)}px`; + }); + $(".hit", svg).addEventListener("mouseleave", () => { hover.setAttribute("visibility", "hidden"); tip.hidden = true; }); +} + +function renderHistory() { + if (!historyData) return; + const { samples, restarts } = historyData; + panel.querySelectorAll(".history-charts .chart").forEach((f) => drawChart(f, samples, restarts, historyHours)); + const crashes = restarts.filter((r) => CRASH_RESETS.has(r.reason)).length; + const plain = restarts.length - crashes; + $(".chart-legend", panel).innerHTML = [ + plain && ` restart${plain > 1 ? `s (${plain})` : ""}`, + crashes && ` ⚠ restart after a crash${crashes > 1 ? `es (${crashes})` : ""}`, + ].filter(Boolean).join(" · ") || "No restart in this period."; +} + +async function loadDeviceHistory(id) { + try { + const data = await api("GET", `/api/devices/${id}/history?hours=${historyHours}`); + if (id !== panelDeviceId) return; + historyData = data; + renderHistory(); + } catch (e) { + toast(e.message, "err"); + } +} + +panel.querySelector(".seg").addEventListener("click", (e) => { + const button = e.target.closest("button[data-hours]"); + if (!button) return; + historyHours = Number(button.dataset.hours); + panel.querySelectorAll(".seg button").forEach((b) => b.setAttribute("aria-pressed", String(b === button))); + loadDeviceHistory(panelDeviceId); +}); +let historyResize; +window.addEventListener("resize", () => { + clearTimeout(historyResize); + historyResize = setTimeout(() => panel.open && renderHistory(), 150); +}); diff --git a/server/otter/static/index.html b/server/otter/static/index.html index cf13063..1fe272e 100644 --- a/server/otter/static/index.html +++ b/server/otter/static/index.html @@ -128,6 +128,21 @@

Connection

+
+
+

History

+
+ + +
+
+
+
Wi-Fi signal dBm
+
Free memory KB
+
+

+
+

Updates

diff --git a/server/otter/static/style.css b/server/otter/static/style.css index 164b088..94111a1 100644 --- a/server/otter/static/style.css +++ b/server/otter/static/style.css @@ -289,6 +289,29 @@ dialog#device-panel::backdrop { background: rgb(10 14 20 / .5); backdrop-filter: .info dd { margin: 2px 0 0; overflow-wrap: anywhere; font-weight: 500; } .info dd .tag { font-size: 12px; } .info dd .radio { display: inline-block; margin: 2px 4px 0 0; padding: 1px 8px; border-radius: 99px; font-size: 12px; background: var(--track); } +.seg { display: inline-flex; padding: 2px; border-radius: 8px; background: var(--track); } +.panel-grid .seg button { border: none; background: none; padding: 3px 10px; font-size: 12px; border-radius: 6px; color: var(--muted); } +.panel-grid .seg button[aria-pressed="true"] { background: var(--surface); color: var(--text); box-shadow: 0 1px 2px rgb(0 0 0 / .12); } +.history-charts { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; margin-top: 10px; } +.chart { margin: 0; min-width: 0; } +.chart figcaption { font-size: 12px; font-weight: 600; margin-bottom: 4px; } +.chart .plot { position: relative; min-height: 150px; } +.chart svg { display: block; width: 100%; height: auto; overflow: visible; } +.chart .grid { stroke: var(--border); stroke-width: 1; } +.chart .axis { fill: var(--muted); font-size: 10px; font-variant-numeric: tabular-nums; } +.chart .line { fill: none; stroke: var(--accent); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +.chart .restart { stroke: var(--muted); stroke-width: 1.5; stroke-dasharray: 3 3; } +.chart .restart.crash { stroke: var(--err); } +.chart .crosshair { stroke: var(--muted); stroke-width: 1; } +.chart .hover circle { fill: var(--accent); stroke: var(--surface); stroke-width: 2; } +.chart .hit { fill: transparent; cursor: crosshair; } +.chart-tip { position: absolute; top: -6px; transform: translateY(-100%); padding: 4px 8px; border-radius: 6px; font-size: 12px; white-space: nowrap; pointer-events: none; background: var(--text); color: var(--bg); } +.chart-tip span { opacity: .75; margin-left: 4px; } +.chart-empty { margin: 0; padding: 50px 0; text-align: center; font-size: 13px; color: var(--muted); } +.chart-legend { margin: 6px 0 0; font-size: 12px; display: flex; align-items: center; gap: 6px; flex-wrap: wrap; } +.chart-legend .key { display: inline-block; width: 0; height: 12px; border-left: 1.5px dashed var(--muted); } +.chart-legend .key.crash { border-color: var(--err); } +@media (max-width: 760px) { .history-charts { grid-template-columns: minmax(0, 1fr); } } .logs-tools { display: flex; align-items: center; gap: 10px; font-size: 12px; } .logs-toggle.on { color: var(--err); } .logs { diff --git a/server/tests/test_history.py b/server/tests/test_history.py new file mode 100644 index 0000000..0205500 --- /dev/null +++ b/server/tests/test_history.py @@ -0,0 +1,67 @@ +"""Signal and memory history (#98).""" + +from datetime import timedelta + +import pytest + +from otter import history +from otter.db import SessionLocal, utcnow +from otter.models import DeviceSample +from test_channels import ids + +MAC = "aa:bb:cc:00:00:01" + + +@pytest.fixture(autouse=True) +def fresh_history(): + history._last_sample.clear() + history._pruned_at = None + yield + + +def get(client, hours=24): + return client.get(f"/api/devices/{ids(client)[MAC]}/history?hours={hours}").json() + + +def test_one_sample_every_five_minutes(client, checkin): + for heap in (200_000, 199_000, 198_000): + checkin(free_heap=heap, uptime_s=100) + samples = get(client)["samples"] + assert [(s["rssi"], s["free_heap"]) for s in samples] == [(-60, 200_000)] + + +def test_a_restart_is_marked_at_once(client, checkin): + checkin(free_heap=200_000, uptime_s=500, boot_count=1, reset_reason="power_on") + checkin(free_heap=210_000, uptime_s=3, boot_count=2, reset_reason="panic") + data = get(client) + assert [s["free_heap"] for s in data["samples"]] == [200_000, 210_000] + assert [r["reason"] for r in data["restarts"]] == ["panic"] + + +def test_old_samples_are_pruned_and_ranges_filter(client, checkin): + checkin(free_heap=200_000) + device_id = ids(client)[MAC] + with SessionLocal() as session: + for days in (1.5, 8): + session.add(DeviceSample(device_id=device_id, at=utcnow() - timedelta(days=days), rssi=-70, free_heap=1)) + session.commit() + assert len(get(client, 24)["samples"]) == 1 + assert len(get(client, 168)["samples"]) == 2 + + def stored(): + with SessionLocal() as session: + return session.query(DeviceSample).count() + + assert stored() == 3 # pruning runs at most hourly + history._pruned_at = None + history._last_sample.clear() + checkin(free_heap=200_000) + assert stored() == 3 # the 8-day-old one is gone, a new one came + + +def test_samples_go_with_the_device(client, checkin): + checkin(free_heap=200_000) + device_id = ids(client)[MAC] + client.delete(f"/api/devices/{device_id}") + with SessionLocal() as session: + assert session.query(DeviceSample).count() == 0