kota-technician-tool/kota_tech/service/influx.py

"""Does the box's data reach InfluxDB? (read-only query, InfluxDB 2.x)

The box writes to the "sensorbox" bucket with the tag device=<MAC without
separators>. device_status and microphone use the system time, the sensor
measurements (AS7341, BME688, SEN66) are timestamped from the box's RTC -
so a box can report its status while none of its sensor data arrives
(e.g. a stopped RTC stamps readings with 1 Jan 2000).

The token must be a read token for the bucket. It comes from the
KOTA_INFLUX_TOKEN environment variable or the tool settings and is never
written to the session log or the service report.
"""

from __future__ import annotations

import csv
import datetime as dt
import io
import json
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field

DEFAULT_URL = "https://influx.kota-app.com"
DEFAULT_ORG = "kota"
DEFAULT_BUCKET = "sensorbox"
LOOKBACK = "-30d"
FRESH_MIN = 15               # like SEEN_OK_MIN in the service system
FUTURE_TOLERANCE_MIN = 10    # newer than now + this: the box clock is ahead
SINCE_SLACK_S = 60           # clock difference box / laptop when counting "since the test started"
TIMEOUT_S = 20

# Sensor groups that every box must upload: name -> measurement prefixes.
SENSOR_GROUPS = {
    "AS7341 (light)": ("as7341_",),
    "BME688 (air)": ("bme688",),
    "SEN66 (air quality)": ("sen66",),
}
STATUS = "device_status"
MIC = "microphone"

OK = "ok"                            # all sensor groups arrive
SENSORS_MISSING = "sensors_missing"  # status/microphone arrive, (some) sensor data does not
SILENT = "silent"                    # data exists, but nothing recent
NEVER = "never"                      # no data for this box in the lookback window
UNAVAILABLE = "unavailable"          # no token / no Internet / query failed
OUTCOMES = (OK, SENSORS_MISSING, SILENT, NEVER, UNAVAILABLE)


@dataclass(frozen=True)
class InfluxConfig:
    token: str
    url: str = DEFAULT_URL
    org: str = DEFAULT_ORG
    bucket: str = DEFAULT_BUCKET


@dataclass
class InfluxCheck:
    outcome: str
    summary: str
    latest: dict[str, dt.datetime] = field(default_factory=dict)   # measurement -> newest point
    missing: list[str] = field(default_factory=list)                # sensor groups without fresh data
    details: list[str] = field(default_factory=list)


def build_query(bucket: str, mac: str) -> str:
    """Newest point per measurement of one box (future times included)."""
    return "\n".join([
        f"from(bucket: {json.dumps(bucket)})",
        f"  |> range(start: {LOOKBACK}, stop: 2100-01-01T00:00:00Z)",
        f"  |> filter(fn: (r) => r.device == {json.dumps(mac)})",
        "  |> last()",
        # Drop _value first: string and float fields cannot share a table.
        '  |> keep(columns: ["_measurement", "_time"])',
        '  |> group(columns: ["_measurement"])',
        '  |> max(column: "_time")',
        "  |> group()",
    ])


def parse_time(value: str) -> dt.datetime:
    """RFC 3339 from InfluxDB; nanoseconds are cut to microseconds."""
    value = value.strip().replace("Z", "+00:00")
    if "." in value:
        head, rest = value.split(".", 1)
        frac, sign, tz = rest.partition("+") if "+" in rest else rest.partition("-")
        value = f"{head}.{frac[:6]}{sign}{tz}"
    return dt.datetime.fromisoformat(value)


def parse_csv(text: str) -> dict[str, dt.datetime]:
    """Annotated/plain CSV from /api/v2/query -> {measurement: newest time}."""
    latest: dict[str, dt.datetime] = {}
    header: list[str] | None = None
    for row in csv.reader(io.StringIO(text)):
        if not row or not any(row):
            header = None          # tables are separated by an empty line
            continue
        if row[0].startswith("#"):
            continue
        if header is None:
            header = row
            continue
        rec = dict(zip(header, row))
        m, t = rec.get("_measurement"), rec.get("_time")
        if m and t:
            ts = parse_time(t)
            if m not in latest or ts > latest[m]:
                latest[m] = ts
    return latest


def _ago(now: dt.datetime, t: dt.datetime) -> str:
    minutes = (now - t).total_seconds() / 60
    if minutes < -1:
        return f"{-minutes:.0f} min in the FUTURE"
    if minutes < 90:
        return f"{max(minutes, 0):.0f} min ago"
    if minutes < 48 * 60:
        return f"{minutes / 60:.0f} h ago"
    return f"{minutes / 1440:.0f} days ago"


def classify(latest: dict[str, dt.datetime], now: dt.datetime, fresh_min: int = FRESH_MIN,
             since: dt.datetime | None = None) -> InfluxCheck:
    """`since`: only data newer than this counts (the box was restarted then)."""
    def newest(prefixes: tuple[str, ...]) -> dt.datetime | None:
        times = [t for m, t in latest.items() if m.startswith(prefixes)]
        return max(times) if times else None

    def fresh(t: dt.datetime | None) -> bool:
        if t is None or (now - t).total_seconds() > fresh_min * 60:
            return False
        return since is None or (since - t).total_seconds() <= SINCE_SLACK_S

    details = []
    for label, prefixes in (("Device status", (STATUS,)), ("Microphone", (MIC,)),
                            *SENSOR_GROUPS.items()):
        t = newest(prefixes)
        details.append(f"{label}: " + (f"{t:%Y-%m-%d %H:%M} UTC ({_ago(now, t)})" if t else "no data"))
        if t is not None and (t - now).total_seconds() > FUTURE_TOLERANCE_MIN * 60:
            details.append(f"{label}: timestamp is in the future - the box clock is wrong")
    missing = [g for g, p in SENSOR_GROUPS.items() if not fresh(newest(p))]
    status_t = max(filter(None, (newest((STATUS,)), newest((MIC,)))), default=None)
    sensors_t = [newest(p) for p in SENSOR_GROUPS.values()]
    lagging = status_t is not None and any(t is None or (status_t - t).total_seconds() > fresh_min * 60
                                           for t in sensors_t)
    if lagging:
        details.append("Sensor data stopped (or never started) while the status still arrived")

    if not latest:
        return InfluxCheck(NEVER, f"No data from this box in InfluxDB in the last {LOOKBACK[1:]}",
                           latest, missing, details)
    if not missing:
        return InfluxCheck(OK, "Sensor data arrives in InfluxDB", latest, missing, details)
    if fresh(status_t):
        return InfluxCheck(SENSORS_MISSING, "Status arrives in InfluxDB, sensor data does not: "
                           + ", ".join(missing), latest, missing, details)
    if lagging:
        # Offline now, but the history already shows the fault.
        return InfluxCheck(SENSORS_MISSING, f"Nothing new in InfluxDB (last status {_ago(now, status_t)}); "
                           "before that the status arrived without sensor data: " + ", ".join(missing),
                           latest, missing, details)
    if since is not None:
        return InfluxCheck(SILENT, "No new data in InfluxDB since the test started", latest, missing, details)
    return InfluxCheck(SILENT, f"No recent data in InfluxDB (nothing in the last {fresh_min} min)",
                       latest, missing, details)


def query_latest(cfg: InfluxConfig, mac: str) -> dict[str, dt.datetime]:
    url = cfg.url.rstrip("/") + "/api/v2/query?" + urllib.parse.urlencode({"org": cfg.org})
    req = urllib.request.Request(url, data=build_query(cfg.bucket, mac).encode(), method="POST", headers={
        "Authorization": f"Token {cfg.token}",
        "Content-Type": "application/vnd.flux",
        "Accept": "application/csv",
    })
    with urllib.request.urlopen(req, timeout=TIMEOUT_S) as resp:
        return parse_csv(resp.read().decode("utf-8"))


def check_device(cfg: InfluxConfig | None, mac: str, now: dt.datetime | None = None,
                 since: dt.datetime | None = None) -> InfluxCheck:
    """Blocking; run it off the UI thread. Never raises."""
    mac = "".join(c for c in (mac or "") if c.isalnum()).upper()
    if cfg is None or not cfg.token:
        return InfluxCheck(UNAVAILABLE, "InfluxDB access is not set up (Settings > InfluxDB access)")
    if len(mac) != 12:
        return InfluxCheck(UNAVAILABLE, "The box MAC address is not known yet")
    try:
        latest = query_latest(cfg, mac)
    except urllib.error.HTTPError as exc:
        why = {401: "the token was rejected", 403: "the token has no read access",
               404: "organisation or bucket not found"}.get(exc.code, f"HTTP {exc.code}")
        return InfluxCheck(UNAVAILABLE, f"InfluxDB query failed: {why}")
    except (urllib.error.URLError, OSError) as exc:
        reason = getattr(exc, "reason", exc)
        return InfluxCheck(UNAVAILABLE, f"InfluxDB not reachable from this computer ({reason})")
    except (ValueError, csv.Error) as exc:
        return InfluxCheck(UNAVAILABLE, f"Unexpected InfluxDB answer ({exc})")
    return classify(latest, now or dt.datetime.now(dt.timezone.utc), since=since)