Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,13 @@ Espressif's `esp_delta_ota`): a few tens of KB instead of a megabyte. The device
really runs the patch's base first, and a retry downloads the whole image. Nothing to configure:
the smallest transfer is picked each time. Other agents keep downloading the plain image.

## History

A device's details page charts its Wi-Fi signal and free memory over the last 24 hours or 7 days,
with its restarts marked (in red after a crash): a memory leak, or a board that drops off at
night, shows at a glance. Otter keeps a sample per device every 5 minutes for 7 days, from the
check-ins (a few MB for a hundred devices).

## Live logs

**Live logs** in a device's details page shows its log output as it runs, without a USB cable:
Expand Down
5 changes: 4 additions & 1 deletion server/otter/api_device.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from sqlalchemy import select
from sqlalchemy.orm import Session

from . import commands, config, crashes, devconfig
from . import commands, config, crashes, devconfig, history
from .auth import token_hash
from .db import SessionLocal, get_session, utcnow
from .device_auth import DeviceAuth, authenticate, check_access, new_token
Expand Down Expand Up @@ -151,6 +151,9 @@ def record_checkin(
if is_new:
notifier.emit("device_new", device_id=device.id)
raise HTTPException(403, "device awaiting approval in Otter")
if device.id is None:
session.flush()
history.record(session, device, restart=(body.reset_reason or "restart") if rebooted else None)
# Until the device uses a token, each fleet-key check-in gets a fresh one.
token = None
if auth.fleet and device.token_used_at is None:
Expand Down
21 changes: 19 additions & 2 deletions server/otter/api_ui.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
"""Endpoints used by the web UI."""

import asyncio
from datetime import UTC
import re

from fastapi import APIRouter, Depends, Form, HTTPException, Request, UploadFile
from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request, UploadFile
from fastapi.responses import FileResponse, StreamingResponse
from sqlalchemy import delete, func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, selectinload

from . import commands, config, crashes, devconfig, signing
from . import commands, config, crashes, devconfig, history, signing
from .api_device import publish_device
from .api_rollouts import reconcile_channels
from .auth import require_user
Expand Down Expand Up @@ -237,9 +238,25 @@ def delete_device(device_id: int, session: Session = Depends(get_session)):
delete_unused_tags(session)
session.commit()
device_logs.forget(device_id)
history.forget(device_id)
broadcaster.publish("device_deleted", {"id": device_id})


# --- History (#98, see history.py) ---------------------------------------------


@router.get("/devices/{device_id}/history")
def device_history(device_id: int, hours: int = Query(24, ge=1, le=168), session: Session = Depends(get_session)):
"""Signal and free memory over the last hours, and the restarts in that time."""
session.get(Device, device_id) or _404("device")
rows = history.samples(session, device_id, hours)
stamp = lambda at: at.replace(tzinfo=UTC).isoformat() # noqa: E731
return {
"samples": [{"t": stamp(s.at), "rssi": s.rssi, "free_heap": s.free_heap} for s in rows],
"restarts": [{"t": stamp(s.at), "reason": s.restart} for s in rows if s.restart],
}


# --- Live logs (#96, see logs.py) ---------------------------------------------


Expand Down
50 changes: 50 additions & 0 deletions server/otter/history.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
"""Signal and free memory history (#98): a sample per device every SAMPLE_EVERY, kept KEEP.

A restart forces a sample at once, carrying its reset reason, so the charts can mark it.
"""

from datetime import timedelta

from sqlalchemy import delete, select
from sqlalchemy.orm import Session

from .db import utcnow
from .models import Device, DeviceSample

SAMPLE_EVERY = timedelta(minutes=5)
KEEP = timedelta(days=7)
PRUNE_EVERY = timedelta(hours=1)

_last_sample: dict[int, object] = {} # device id -> time of its last sample, this process
_pruned_at = None


def record(session: Session, device: Device, restart: str | None = None) -> None:
"""Called at each check-in; writes a sample when one is due."""
global _pruned_at
now = utcnow()
last = _last_sample.get(device.id)
if restart is None and (last is not None and now - last < SAMPLE_EVERY):
return
if restart is None and device.rssi is None and device.free_heap is None:
return
session.add(DeviceSample(device_id=device.id, at=now, rssi=device.rssi, free_heap=device.free_heap, restart=restart))
_last_sample[device.id] = now
if _pruned_at is None or now - _pruned_at > PRUNE_EVERY:
session.execute(delete(DeviceSample).where(DeviceSample.at < now - KEEP))
_pruned_at = now


def samples(session: Session, device_id: int, hours: int) -> list[DeviceSample]:
since = utcnow() - timedelta(hours=hours)
return list(
session.scalars(
select(DeviceSample)
.where(DeviceSample.device_id == device_id, DeviceSample.at >= since)
.order_by(DeviceSample.at)
)
)


def forget(device_id: int) -> None:
_last_sample.pop(device_id, None)
39 changes: 39 additions & 0 deletions server/otter/migrations/versions/0021_device_samples.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""device samples

Revision ID: 0021
Revises: 0020
Create Date: 2026-10-01 22:00:00.000000
"""

from collections.abc import Sequence

import sqlalchemy as sa
from alembic import op


revision: str = '0021'
down_revision: str | None = '0020'
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None


def upgrade() -> None:
op.create_table(
'device_samples',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('device_id', sa.Integer(), nullable=False),
sa.Column('at', sa.DateTime(), nullable=False),
sa.Column('rssi', sa.Integer(), nullable=True),
sa.Column('free_heap', sa.Integer(), nullable=True),
sa.Column('restart', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['device_id'], ['devices.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
)
op.create_index('ix_device_samples_at', 'device_samples', ['at'])
op.create_index('ix_device_samples_device_at', 'device_samples', ['device_id', 'at'])


def downgrade() -> None:
op.drop_index('ix_device_samples_device_at', table_name='device_samples')
op.drop_index('ix_device_samples_at', table_name='device_samples')
op.drop_table('device_samples')
18 changes: 17 additions & 1 deletion server/otter/models.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import json
from datetime import datetime

from sqlalchemy import Column, ForeignKey, String, Table, UniqueConstraint
from sqlalchemy import Column, ForeignKey, Index, String, Table, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship

from .db import Base, utcnow
Expand Down Expand Up @@ -253,6 +253,22 @@ class ConfigValue(Base):
value: Mapped[str] # JSON: a string, number or boolean


class DeviceSample(Base):
"""A device's signal and free memory at one check-in (#98): one every few minutes, kept a
week (see history.py). restart is the reset reason when the device had just restarted."""

__tablename__ = "device_samples"

id: Mapped[int] = mapped_column(primary_key=True)
device_id: Mapped[int] = mapped_column(ForeignKey("devices.id", ondelete="CASCADE"))
at: Mapped[datetime] = mapped_column(default=utcnow, index=True)
rssi: Mapped[int | None]
free_heap: Mapped[int | None]
restart: Mapped[str | None]

__table_args__ = (Index("ix_device_samples_device_at", "device_id", "at"),)


class Crash(Base):
"""A crash report (#24): the core dump summary a device sent after restarting from a crash."""

Expand Down
136 changes: 136 additions & 0 deletions server/otter/static/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,9 @@ function openPanel(id) {
panel.focus(); // not its first button, which would show a focus ring
loadCommands(id);
loadLogs(id);
historyData = null;
panel.querySelectorAll(".history-charts .plot").forEach((p) => { p.innerHTML = ""; });
loadDeviceHistory(id);
loadConfig(id, "device");
loadCrashes(id);
}
Expand Down Expand Up @@ -1417,3 +1420,136 @@ $(".logs-toggle", panel).addEventListener("click", () => {
watchLogs();
logsRenew = setInterval(watchLogs, LOGS_RENEW_MS);
});

// --- History charts (#98) ------------------------------------------------------
// A sample every 5 minutes: signal and free memory, one chart each (never two scales on one),
// restarts marked. Hovering shows the nearest sample.

let historyHours = 24;
let historyData = null;
const SAMPLE_GAP_MS = 16 * 60 * 1000; // more than 3 missed samples: the device was away
const CHARTS = {
rssi: { value: (s) => s.rssi, format: (v) => `${v} dBm` },
free_heap: { value: (s) => (s.free_heap == null ? null : s.free_heap / 1024), format: (v) => `${Math.round(v)} KB` },
};

function niceTicks(min, max, count = 3) {
const span = max - min || 1;
const step = [1, 2, 5, 10, 20, 25, 50, 100, 200, 250, 500, 1000].find((s) => span / s <= count) || span / count;
const ticks = [];
for (let v = Math.ceil(min / step) * step; v <= max; v += step) ticks.push(v);
return ticks;
}

// Over a week, 5-minute samples are noise: their mean per bucket (a gap still breaks the line).
function averaged(points, bucketMs) {
const buckets = new Map();
for (const p of points) {
const key = Math.floor(p.t / bucketMs);
const b = buckets.get(key) || buckets.set(key, { t: 0, v: 0, n: 0 }).get(key);
b.t += p.t; b.v += p.v; b.n += 1;
}
return [...buckets.values()].map((b) => ({ t: b.t / b.n, v: b.v / b.n }));
}

function drawChart(figure, samples, restarts, hours) {
const plot = $(".plot", figure);
const { value, format } = CHARTS[figure.dataset.key];
let points = samples.map((s) => ({ t: Date.parse(s.t), v: value(s) })).filter((p) => p.v != null);
if (hours > 24) points = averaged(points, 30 * 60 * 1000);
if (points.length < 2) {
plot.innerHTML = `<p class="chart-empty">${points.length ? "One sample so far" : "No samples yet"}: one comes every 5 minutes.</p>`;
return;
}
const W = Math.max(240, plot.clientWidth), H = 150, L = 40, R = 8, T = 8, B = 22;
const end = Date.now(), start = end - hours * 3600 * 1000;
let lo = Math.min(...points.map((p) => p.v)), hi = Math.max(...points.map((p) => p.v));
const pad = (hi - lo) * 0.15 || Math.abs(hi) * 0.05 || 1;
lo -= pad; hi += pad;
const x = (t) => L + ((t - start) / (end - start)) * (W - L - R);
const y = (v) => T + (1 - (v - lo) / (hi - lo)) * (H - T - B);

// Grid and axes: recessive.
const yTicks = niceTicks(lo, hi).map((v) =>
`<line class="grid" x1="${L}" x2="${W - R}" y1="${y(v)}" y2="${y(v)}"/><text class="axis" x="${L - 6}" y="${y(v) + 4}" text-anchor="end">${Math.round(v)}</text>`);
const stepH = hours <= 24 ? 6 : 24;
const xTicks = [];
const first = new Date(start); first.setMinutes(0, 0, 0);
for (let t = first.getTime() + 3600e3; t < end; t += 3600e3) {
const d = new Date(t);
if (d.getHours() % stepH) continue;
const text = stepH === 24 ? d.toLocaleDateString([], { weekday: "short" }) : `${String(d.getHours()).padStart(2, "0")}:00`;
xTicks.push(`<text class="axis" x="${x(t)}" y="${H - 6}" text-anchor="middle">${text}</text>`);
}
// The line, broken where the device was away.
let path = "";
points.forEach((p, i) => {
const gap = i === 0 || p.t - points[i - 1].t > (hours > 24 ? 2.5 * 30 * 60 * 1000 : SAMPLE_GAP_MS);
path += `${gap ? "M" : "L"}${x(p.t).toFixed(1)},${y(p.v).toFixed(1)}`;
});
const marks = restarts.map((r) => {
const crash = CRASH_RESETS.has(r.reason);
const rx = x(Date.parse(r.t));
return `<line class="restart${crash ? " crash" : ""}" x1="${rx}" x2="${rx}" y1="${T}" y2="${H - B}"><title>Restarted: ${esc(r.reason.replaceAll("_", " "))}</title></line>`;
});
plot.innerHTML = `
<svg viewBox="0 0 ${W} ${H}" width="${W}" height="${H}" role="img"
aria-label="${esc($("figcaption", figure).textContent)}: from ${format(Math.min(...points.map((p) => p.v)))} to ${format(Math.max(...points.map((p) => p.v)))}, last ${format(points.at(-1).v)}">
${yTicks.join("")}${xTicks.join("")}${marks.join("")}
<path class="line" d="${path}"/>
<g class="hover" visibility="hidden"><line class="crosshair" y1="${T}" y2="${H - B}"/><circle r="4"/></g>
<rect class="hit" x="${L}" y="0" width="${W - L - R}" height="${H}"/>
</svg>
<div class="chart-tip" hidden></div>`;
const svg = $("svg", plot), hover = $(".hover", svg), tip = $(".chart-tip", plot);
$(".hit", svg).addEventListener("mousemove", (e) => {
const box = svg.getBoundingClientRect();
const t = start + ((e.clientX - box.left) * (W / box.width) - L) / (W - L - R) * (end - start);
const p = points.reduce((a, b) => (Math.abs(b.t - t) < Math.abs(a.t - t) ? b : a));
const px = x(p.t), py = y(p.v);
hover.setAttribute("visibility", "visible");
$(".crosshair", hover).setAttribute("x1", px); $(".crosshair", hover).setAttribute("x2", px);
$("circle", hover).setAttribute("cx", px); $("circle", hover).setAttribute("cy", py);
const when = new Date(p.t).toLocaleString([], { weekday: hours > 24 ? "short" : undefined, hour: "2-digit", minute: "2-digit" });
tip.innerHTML = `<strong>${format(p.v)}</strong> <span>${when}</span>`;
tip.hidden = false;
tip.style.left = `${Math.min(px * (box.width / W), box.width - tip.offsetWidth)}px`;
});
$(".hit", svg).addEventListener("mouseleave", () => { hover.setAttribute("visibility", "hidden"); tip.hidden = true; });
}

function renderHistory() {
if (!historyData) return;
const { samples, restarts } = historyData;
panel.querySelectorAll(".history-charts .chart").forEach((f) => drawChart(f, samples, restarts, historyHours));
const crashes = restarts.filter((r) => CRASH_RESETS.has(r.reason)).length;
const plain = restarts.length - crashes;
$(".chart-legend", panel).innerHTML = [
plain && `<span class="key"></span> restart${plain > 1 ? `s (${plain})` : ""}`,
crashes && `<span class="key crash"></span> ⚠ restart after a crash${crashes > 1 ? `es (${crashes})` : ""}`,
].filter(Boolean).join(" · ") || "No restart in this period.";
}

async function loadDeviceHistory(id) {
try {
const data = await api("GET", `/api/devices/${id}/history?hours=${historyHours}`);
if (id !== panelDeviceId) return;
historyData = data;
renderHistory();
} catch (e) {
toast(e.message, "err");
}
}

panel.querySelector(".seg").addEventListener("click", (e) => {
const button = e.target.closest("button[data-hours]");
if (!button) return;
historyHours = Number(button.dataset.hours);
panel.querySelectorAll(".seg button").forEach((b) => b.setAttribute("aria-pressed", String(b === button)));
loadDeviceHistory(panelDeviceId);
});
let historyResize;
window.addEventListener("resize", () => {
clearTimeout(historyResize);
historyResize = setTimeout(() => panel.open && renderHistory(), 150);
});
15 changes: 15 additions & 0 deletions server/otter/static/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,21 @@ <h3>Connection</h3>
<dl class="info info-net"></dl>
</section>

<section class="pcard wide">
<div class="pcard-head">
<h3>History</h3>
<div class="seg" role="group" aria-label="Time range">
<button type="button" data-hours="24" aria-pressed="true">24 h</button>
<button type="button" data-hours="168" aria-pressed="false">7 days</button>
</div>
</div>
<div class="history-charts">
<figure class="chart" data-key="rssi"><figcaption>Wi-Fi signal <span class="muted">dBm</span></figcaption><div class="plot"></div></figure>
<figure class="chart" data-key="free_heap"><figcaption>Free memory <span class="muted">KB</span></figcaption><div class="plot"></div></figure>
</div>
<p class="chart-legend muted"></p>
</section>

<section class="pcard wide">
<div class="pcard-head">
<h3>Updates</h3>
Expand Down
23 changes: 23 additions & 0 deletions server/otter/static/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,29 @@ dialog#device-panel::backdrop { background: rgb(10 14 20 / .5); backdrop-filter:
.info dd { margin: 2px 0 0; overflow-wrap: anywhere; font-weight: 500; }
.info dd .tag { font-size: 12px; }
.info dd .radio { display: inline-block; margin: 2px 4px 0 0; padding: 1px 8px; border-radius: 99px; font-size: 12px; background: var(--track); }
.seg { display: inline-flex; padding: 2px; border-radius: 8px; background: var(--track); }
.panel-grid .seg button { border: none; background: none; padding: 3px 10px; font-size: 12px; border-radius: 6px; color: var(--muted); }
.panel-grid .seg button[aria-pressed="true"] { background: var(--surface); color: var(--text); box-shadow: 0 1px 2px rgb(0 0 0 / .12); }
.history-charts { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; margin-top: 10px; }
.chart { margin: 0; min-width: 0; }
.chart figcaption { font-size: 12px; font-weight: 600; margin-bottom: 4px; }
.chart .plot { position: relative; min-height: 150px; }
.chart svg { display: block; width: 100%; height: auto; overflow: visible; }
.chart .grid { stroke: var(--border); stroke-width: 1; }
.chart .axis { fill: var(--muted); font-size: 10px; font-variant-numeric: tabular-nums; }
.chart .line { fill: none; stroke: var(--accent); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; }
.chart .restart { stroke: var(--muted); stroke-width: 1.5; stroke-dasharray: 3 3; }
.chart .restart.crash { stroke: var(--err); }
.chart .crosshair { stroke: var(--muted); stroke-width: 1; }
.chart .hover circle { fill: var(--accent); stroke: var(--surface); stroke-width: 2; }
.chart .hit { fill: transparent; cursor: crosshair; }
.chart-tip { position: absolute; top: -6px; transform: translateY(-100%); padding: 4px 8px; border-radius: 6px; font-size: 12px; white-space: nowrap; pointer-events: none; background: var(--text); color: var(--bg); }
.chart-tip span { opacity: .75; margin-left: 4px; }
.chart-empty { margin: 0; padding: 50px 0; text-align: center; font-size: 13px; color: var(--muted); }
.chart-legend { margin: 6px 0 0; font-size: 12px; display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
.chart-legend .key { display: inline-block; width: 0; height: 12px; border-left: 1.5px dashed var(--muted); }
.chart-legend .key.crash { border-color: var(--err); }
@media (max-width: 760px) { .history-charts { grid-template-columns: minmax(0, 1fr); } }
.logs-tools { display: flex; align-items: center; gap: 10px; font-size: 12px; }
.logs-toggle.on { color: var(--err); }
.logs {
Expand Down
Loading
Loading