diff --git a/README.md b/README.md index f1144e3..fe6c648 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,13 @@ Espressif's `esp_delta_ota`): a few tens of KB instead of a megabyte. The device really runs the patch's base first, and a retry downloads the whole image. Nothing to configure: the smallest transfer is picked each time. Other agents keep downloading the plain image. +## History + +A device's details page charts its Wi-Fi signal and free memory over the last 24 hours or 7 days, +with its restarts marked (in red after a crash): a memory leak, or a board that drops off at +night, shows at a glance. Otter keeps a sample per device every 5 minutes for 7 days, from the +check-ins (a few MB for a hundred devices). + ## Live logs **Live logs** in a device's details page shows its log output as it runs, without a USB cable: diff --git a/server/otter/api_device.py b/server/otter/api_device.py index 7909906..1f77eaa 100644 --- a/server/otter/api_device.py +++ b/server/otter/api_device.py @@ -12,7 +12,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from . import commands, config, crashes, devconfig +from . import commands, config, crashes, devconfig, history from .auth import token_hash from .db import SessionLocal, get_session, utcnow from .device_auth import DeviceAuth, authenticate, check_access, new_token @@ -151,6 +151,9 @@ def record_checkin( if is_new: notifier.emit("device_new", device_id=device.id) raise HTTPException(403, "device awaiting approval in Otter") + if device.id is None: + session.flush() + history.record(session, device, restart=(body.reset_reason or "restart") if rebooted else None) # Until the device uses a token, each fleet-key check-in gets a fresh one. token = None if auth.fleet and device.token_used_at is None: diff --git a/server/otter/api_ui.py b/server/otter/api_ui.py index e251d40..6a89c24 100644 --- a/server/otter/api_ui.py +++ b/server/otter/api_ui.py @@ -1,15 +1,16 @@ """Endpoints used by the web UI.""" import asyncio +from datetime import UTC import re -from fastapi import APIRouter, Depends, Form, HTTPException, Request, UploadFile +from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import FileResponse, StreamingResponse from sqlalchemy import delete, func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, selectinload -from . import commands, config, crashes, devconfig, signing +from . import commands, config, crashes, devconfig, history, signing from .api_device import publish_device from .api_rollouts import reconcile_channels from .auth import require_user @@ -237,9 +238,25 @@ def delete_device(device_id: int, session: Session = Depends(get_session)): delete_unused_tags(session) session.commit() device_logs.forget(device_id) + history.forget(device_id) broadcaster.publish("device_deleted", {"id": device_id}) +# --- History (#98, see history.py) --------------------------------------------- + + +@router.get("/devices/{device_id}/history") +def device_history(device_id: int, hours: int = Query(24, ge=1, le=168), session: Session = Depends(get_session)): + """Signal and free memory over the last hours, and the restarts in that time.""" + session.get(Device, device_id) or _404("device") + rows = history.samples(session, device_id, hours) + stamp = lambda at: at.replace(tzinfo=UTC).isoformat() # noqa: E731 + return { + "samples": [{"t": stamp(s.at), "rssi": s.rssi, "free_heap": s.free_heap} for s in rows], + "restarts": [{"t": stamp(s.at), "reason": s.restart} for s in rows if s.restart], + } + + # --- Live logs (#96, see logs.py) --------------------------------------------- diff --git a/server/otter/history.py b/server/otter/history.py new file mode 100644 index 0000000..5753403 --- /dev/null +++ b/server/otter/history.py @@ -0,0 +1,50 @@ +"""Signal and free memory history (#98): a sample per device every SAMPLE_EVERY, kept KEEP. + +A restart forces a sample at once, carrying its reset reason, so the charts can mark it. +""" + +from datetime import timedelta + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from .db import utcnow +from .models import Device, DeviceSample + +SAMPLE_EVERY = timedelta(minutes=5) +KEEP = timedelta(days=7) +PRUNE_EVERY = timedelta(hours=1) + +_last_sample: dict[int, object] = {} # device id -> time of its last sample, this process +_pruned_at = None + + +def record(session: Session, device: Device, restart: str | None = None) -> None: + """Called at each check-in; writes a sample when one is due.""" + global _pruned_at + now = utcnow() + last = _last_sample.get(device.id) + if restart is None and (last is not None and now - last < SAMPLE_EVERY): + return + if restart is None and device.rssi is None and device.free_heap is None: + return + session.add(DeviceSample(device_id=device.id, at=now, rssi=device.rssi, free_heap=device.free_heap, restart=restart)) + _last_sample[device.id] = now + if _pruned_at is None or now - _pruned_at > PRUNE_EVERY: + session.execute(delete(DeviceSample).where(DeviceSample.at < now - KEEP)) + _pruned_at = now + + +def samples(session: Session, device_id: int, hours: int) -> list[DeviceSample]: + since = utcnow() - timedelta(hours=hours) + return list( + session.scalars( + select(DeviceSample) + .where(DeviceSample.device_id == device_id, DeviceSample.at >= since) + .order_by(DeviceSample.at) + ) + ) + + +def forget(device_id: int) -> None: + _last_sample.pop(device_id, None) diff --git a/server/otter/migrations/versions/0021_device_samples.py b/server/otter/migrations/versions/0021_device_samples.py new file mode 100644 index 0000000..5032451 --- /dev/null +++ b/server/otter/migrations/versions/0021_device_samples.py @@ -0,0 +1,39 @@ +"""device samples + +Revision ID: 0021 +Revises: 0020 +Create Date: 2026-10-01 22:00:00.000000 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + + +revision: str = '0021' +down_revision: str | None = '0020' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + 'device_samples', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('device_id', sa.Integer(), nullable=False), + sa.Column('at', sa.DateTime(), nullable=False), + sa.Column('rssi', sa.Integer(), nullable=True), + sa.Column('free_heap', sa.Integer(), nullable=True), + sa.Column('restart', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['device_id'], ['devices.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_device_samples_at', 'device_samples', ['at']) + op.create_index('ix_device_samples_device_at', 'device_samples', ['device_id', 'at']) + + +def downgrade() -> None: + op.drop_index('ix_device_samples_device_at', table_name='device_samples') + op.drop_index('ix_device_samples_at', table_name='device_samples') + op.drop_table('device_samples') diff --git a/server/otter/models.py b/server/otter/models.py index f084c57..59dad75 100644 --- a/server/otter/models.py +++ b/server/otter/models.py @@ -1,7 +1,7 @@ import json from datetime import datetime -from sqlalchemy import Column, ForeignKey, String, Table, UniqueConstraint +from sqlalchemy import Column, ForeignKey, Index, String, Table, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from .db import Base, utcnow @@ -253,6 +253,22 @@ class ConfigValue(Base): value: Mapped[str] # JSON: a string, number or boolean +class DeviceSample(Base): + """A device's signal and free memory at one check-in (#98): one every few minutes, kept a + week (see history.py). restart is the reset reason when the device had just restarted.""" + + __tablename__ = "device_samples" + + id: Mapped[int] = mapped_column(primary_key=True) + device_id: Mapped[int] = mapped_column(ForeignKey("devices.id", ondelete="CASCADE")) + at: Mapped[datetime] = mapped_column(default=utcnow, index=True) + rssi: Mapped[int | None] + free_heap: Mapped[int | None] + restart: Mapped[str | None] + + __table_args__ = (Index("ix_device_samples_device_at", "device_id", "at"),) + + class Crash(Base): """A crash report (#24): the core dump summary a device sent after restarting from a crash.""" diff --git a/server/otter/static/app.js b/server/otter/static/app.js index c3b5349..7c98c9a 100644 --- a/server/otter/static/app.js +++ b/server/otter/static/app.js @@ -525,6 +525,9 @@ function openPanel(id) { panel.focus(); // not its first button, which would show a focus ring loadCommands(id); loadLogs(id); + historyData = null; + panel.querySelectorAll(".history-charts .plot").forEach((p) => { p.innerHTML = ""; }); + loadDeviceHistory(id); loadConfig(id, "device"); loadCrashes(id); } @@ -1417,3 +1420,136 @@ $(".logs-toggle", panel).addEventListener("click", () => { watchLogs(); logsRenew = setInterval(watchLogs, LOGS_RENEW_MS); }); + +// --- History charts (#98) ------------------------------------------------------ +// A sample every 5 minutes: signal and free memory, one chart each (never two scales on one), +// restarts marked. Hovering shows the nearest sample. + +let historyHours = 24; +let historyData = null; +const SAMPLE_GAP_MS = 16 * 60 * 1000; // more than 3 missed samples: the device was away +const CHARTS = { + rssi: { value: (s) => s.rssi, format: (v) => `${v} dBm` }, + free_heap: { value: (s) => (s.free_heap == null ? null : s.free_heap / 1024), format: (v) => `${Math.round(v)} KB` }, +}; + +function niceTicks(min, max, count = 3) { + const span = max - min || 1; + const step = [1, 2, 5, 10, 20, 25, 50, 100, 200, 250, 500, 1000].find((s) => span / s <= count) || span / count; + const ticks = []; + for (let v = Math.ceil(min / step) * step; v <= max; v += step) ticks.push(v); + return ticks; +} + +// Over a week, 5-minute samples are noise: their mean per bucket (a gap still breaks the line). +function averaged(points, bucketMs) { + const buckets = new Map(); + for (const p of points) { + const key = Math.floor(p.t / bucketMs); + const b = buckets.get(key) || buckets.set(key, { t: 0, v: 0, n: 0 }).get(key); + b.t += p.t; b.v += p.v; b.n += 1; + } + return [...buckets.values()].map((b) => ({ t: b.t / b.n, v: b.v / b.n })); +} + +function drawChart(figure, samples, restarts, hours) { + const plot = $(".plot", figure); + const { value, format } = CHARTS[figure.dataset.key]; + let points = samples.map((s) => ({ t: Date.parse(s.t), v: value(s) })).filter((p) => p.v != null); + if (hours > 24) points = averaged(points, 30 * 60 * 1000); + if (points.length < 2) { + plot.innerHTML = `
${points.length ? "One sample so far" : "No samples yet"}: one comes every 5 minutes.
`; + return; + } + const W = Math.max(240, plot.clientWidth), H = 150, L = 40, R = 8, T = 8, B = 22; + const end = Date.now(), start = end - hours * 3600 * 1000; + let lo = Math.min(...points.map((p) => p.v)), hi = Math.max(...points.map((p) => p.v)); + const pad = (hi - lo) * 0.15 || Math.abs(hi) * 0.05 || 1; + lo -= pad; hi += pad; + const x = (t) => L + ((t - start) / (end - start)) * (W - L - R); + const y = (v) => T + (1 - (v - lo) / (hi - lo)) * (H - T - B); + + // Grid and axes: recessive. + const yTicks = niceTicks(lo, hi).map((v) => + `