From bfa7edce6fb486bebed4fa4b70b078dee6d0a240 Mon Sep 17 00:00:00 2001 From: Krzysztof Kallas Date: Mon, 28 Sep 2026 19:19:55 +0200 Subject: [PATCH] Technician tool v0.8 Imported from the Google Drive folder 004-technician_tool/v0.8. --- README.md | 41 +++ docs/CHANGES-v0.8.md | 21 ++ docs/mac_agent_command.txt | 65 ++--- kota_tech/__init__.py | 2 +- kota_tech/diag/rules.toml | 8 + kota_tech/sd/provision.py | 521 +++++++++++++++++++------------------ pyproject.toml | 2 +- run_macos.command | 4 +- run_windows.bat | 2 +- tests/test_engine.py | 10 + tests/test_ports.py | 12 + tests/test_sd.py | 11 + uv.lock | 2 +- 13 files changed, 406 insertions(+), 295 deletions(-) create mode 100644 README.md create mode 100644 docs/CHANGES-v0.8.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..1c3ba68 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# KOTA Sensorbox Technician Tool v0.8 + +Diagnoza sensorboxa przez USB, wgrywanie firmware z folderu `firmware/` i przygotowanie karty SD. +Działa na Windows 10/11 i macOS 13 (Ventura) lub nowszym, na Macach z Intelem i Apple Silicon. + +## Uruchomienie + +**Windows:** dwuklik `run_windows.bat`. + +**macOS:** dwuklik `run_macos.command`. +Jeśli Finder odpowie, że pliku nie można uruchomić (Google Drive gubi uprawnienie „wykonywalny”), +otwórz Terminal i jednorazowo wpisz `bash ` (ze spacją), przeciągnij plik `run_macos.command` do okna i naciśnij Enter. +Na stałe naprawia to: `chmod +x run_macos.command`. + +Przy pierwszym uruchomieniu skrypt proponuje instalację `uv` (menedżer Pythona, instaluje się tylko dla bieżącego +użytkownika) i pobiera biblioteki. Potrzebny jest internet, trwa to 1–2 minuty. Kolejne starty są natychmiastowe. +Środowisko jest poza folderem Google Drive: +`%LOCALAPPDATA%\KotaTechnicianTool\venv-0.8` (Windows), `~/Library/Caches/KotaTechnicianTool/venv-0.8` (macOS). + +Sesje, kopie zapasowe flash i ustawienia są w `~/KotaTechnicianTool` (Windows: `C:\Users\\KotaTechnicianTool`). + +## Port sensorboxa + +- Windows: `COMx - Sensorbox (native USB) [MAC]` +- macOS: `/dev/cu.usbmodemXXXX - Sensorbox (native USB) [MAC]` + +Kabel USB musi przenosić dane (nie tylko ładowanie). Przy restartach z komunikatem o brownout użyj innego portu +lub zasilacza. + +## Firmware + +Wgrać można tylko pliki `.bin` z folderu `firmware/`. Lista pokazuje datę kompilacji zapisaną w pliku, +najnowszy build jest na górze. Nowy build: wrzuć plik do `firmware/` i kliknij **Refresh list**. + +## Testy (dla zespołu) + +``` +uv run --frozen pytest -q +``` + +Zmiany: `docs/CHANGES-v0.7.md`, `docs/CHANGES-v0.8.md`. diff --git a/docs/CHANGES-v0.8.md b/docs/CHANGES-v0.8.md new file mode 100644 index 0000000..7a47cf7 --- /dev/null +++ b/docs/CHANGES-v0.8.md @@ -0,0 +1,21 @@ +# Technician tool v0.8 - changes + +## Windows + macOS +- Dependencies verified to install on macOS arm64 and x86_64 (uv.lock: PySide6 6.11 universal2 wheels, + requires macOS 13+); Windows unchanged. +- SD card check: macOS `diskutil` reports FAT32 partitions as `DOS_FAT_32` / `Windows_FAT_32`; these are now + accepted (previously a correct card was reported as "not FAT32" on a Mac). +- Unit tests for the macOS serial-port filter (`/dev/cu.*` only) and FAT32 names. +- README.md with launch steps for both systems (incl. the Google Drive executable-bit workaround on macOS); + docs/mac_agent_command.txt updated for a v0.8 test run on a Mac. + +## Diagnosis +- Harmless Wi-Fi start-up errors are ignored (rule `benign.wifi_start`): `wifi:sta is connecting, return error`, + `esp_wifi_connect() failed: ESP_ERR_WIFI_CONN`, `Failed to get configured network when updating network + status` (unpaired box). +- Brown-out resets are detected (`E BOD: Brownout detector was triggered`, ROM reports RTC_SW_SYS_RST). +- Service report note = the Diagnosis panel as text (LED, firmware, box self-report, start-up progress with + times, findings with "Seen Nx, first at ... s, last at ... s" and log line), shortened to fit the service + form's 5000 characters. + +Not yet run on a real Mac: see docs/mac_agent_command.txt. diff --git a/docs/mac_agent_command.txt b/docs/mac_agent_command.txt index cf5ae64..6068fb0 100644 --- a/docs/mac_agent_command.txt +++ b/docs/mac_agent_command.txt @@ -1,30 +1,35 @@ -You are testing the KOTA sensorbox technician tool v0.5 on macOS. The current folder is -004-technician_tool/v0.5 (a Python/PySide6 desktop app; it has only been tested on Windows so far). - -Rules: -- Python only via uv or conda/mamba. Never use bare/system python or pip. -- Do not build, modify or flash firmware other than the bundled firmware/new_system.bin. -- Do not push anything to Git or GitLab. This is a local demo. -- Ask me before any destructive action: formatting an SD card, Full-mode flashing, or deleting files. - Before formatting, show me the exact disk (identifier, name, size, current files). -- Do not change code without telling me. Report bugs with evidence (commands and output). - -Tests, in order. Report pass/fail with evidence for each: -1. Launch: `bash run_macos.command` (it offers to install uv). The app window opens; note any warnings. -2. Tests: `UV_PROJECT_ENVIRONMENT="$HOME/Library/Caches/KotaTechnicianTool/venv-0.5" uv run --frozen pytest -q` -3. Serial: plug in the sensorbox (native USB). The port list shows a /dev/cu.usbmodem* "Sensorbox (native USB)" - entry with a MAC. Connect, press "Reset & capture": the boot log starts at "ESP-ROM:esp32s3" and the - Diagnosis panel identifies firmware v1.0 (build 45fa9d5b9). Check whether the port disappears after the - reset and whether the tool reconnects. -4. Guided troubleshooting: run "Quick check" and answer truthfully with what I tell you the LED shows. -5. Flash (App only, factory image; the box already runs it): the tool must enter flashing mode by itself, - write, restart the box and report "Success ... build 45fa9d5b9". Capture the esptool output. -6. SD card tab: the settings load from ../../002-SD_card_format/ENV.txt. The card reader and card are listed - (internal disks must NOT be listed). Run "Check card", then, after my OK, "Format card and write ENV.TXT". - It must use `diskutil eraseDisk FAT32 KOTA MBRFormat` on the correct disk, write ENV.TXT byte-identical - to the template, and eject the card. -7. "Service report..." opens the browser at https://service.kota-app.com/#kota-report=... (it is fine - if the page itself cannot prefill yet). - -At the end, write a short report: macOS version, Mac model/architecture, each test with result, errors, and -suggested fixes (file:line). \ No newline at end of file +You are testing the KOTA sensorbox technician tool v0.8 on macOS. The current folder is +004-technician_tool/v0.8 (a Python/PySide6 desktop app; tested on Windows 11, macOS paths are covered by +unit tests but not yet run on a real Mac). + +Rules: +- Python only via uv or conda/mamba. Never use bare/system python or pip. +- Do not build or modify firmware. Flash only files from the tool's firmware/ folder. +- Do not push anything to Git or GitLab. This is a local demo. +- Ask me before any destructive action: formatting an SD card, Full-mode flashing, or deleting files. + Before formatting, show me the exact disk (identifier, name, size, current files). +- Do not change code without telling me. Report bugs with evidence (commands and output). + +Tests, in order. Report pass/fail with evidence for each: +1. Launch: `bash run_macos.command` (it offers to install uv). The app window opens; note any warnings. + Also check whether a double-click in Finder works (Google Drive may drop the executable bit). +2. Tests: `UV_PROJECT_ENVIRONMENT="$HOME/Library/Caches/KotaTechnicianTool/venv-0.8" uv run --frozen pytest -q` + (expect 172 passed). +3. Serial: plug in the sensorbox (native USB). The port list shows a /dev/cu.usbmodem* "Sensorbox (native USB)" + entry with a MAC (no /dev/tty.* duplicate). Connect, press "Reset & capture": the log starts at + "ESP-ROM:esp32s3" and the Diagnosis panel shows firmware v1.0.16 (build 4f10989e), the "Box self-report" + section and the LED with its pattern. Check whether the port disappears after the reset and whether the + tool reconnects. Wait 2 minutes: a "KOTA: health" line appears every 60 s. +4. Show filter "KOTA diagnostics + errors" shows only KOTA lines and errors. +5. Guided troubleshooting: run "Quick check" and answer truthfully with what I tell you the LED shows. +6. Flash (App only, the newest file in firmware/, chosen by its compile date): the tool must enter flashing mode + by itself, write, restart the box and report "Success ... build 4f10989e". Capture the esptool output. +7. SD card tab: the card reader and card are listed (internal disks must NOT be listed). "Check card" must NOT + report a wrong file system for a FAT32 card. After my OK, "Format card and write ENV.TXT" must use + `diskutil eraseDisk FAT32 KOTA MBRFormat` on the correct disk, write ENV.TXT and eject the card. +8. "Service report..." opens the browser at https://service.kota-app.com/#kota-report=...; the note in + the form must match the Diagnosis panel (self-report, start-up progress with times, findings with + "Seen Nx, first at ... s"). + +At the end, write a short report: macOS version, Mac model/architecture, each test with result, errors, and +suggested fixes (file:line). diff --git a/kota_tech/__init__.py b/kota_tech/__init__.py index 2376837..0b261f5 100644 --- a/kota_tech/__init__.py +++ b/kota_tech/__init__.py @@ -1,3 +1,3 @@ """KOTA sensorbox technician tool.""" -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/kota_tech/diag/rules.toml b/kota_tech/diag/rules.toml index 1792c35..0e70f1a 100644 --- a/kota_tech/diag/rules.toml +++ b/kota_tech/diag/rules.toml @@ -137,6 +137,14 @@ id = "benign.zcl" pattern = 'chip\[ZCL\]: WRITE ERR|data_model: Cluster cannot be NULL' severity = "ignore" +[[rule]] +id = "benign.wifi_start" +# Harmless Wi-Fi noise seen on every 1.0.16 boot: Matter asks for a connect +# while the station is already connecting, and an unpaired box polls a +# network it does not have yet (matter.not_commissioned covers that case). +pattern = 'wifi:sta is connecting, return error|esp_wifi_connect\(\) failed: ESP_ERR_WIFI_CONN|Failed to get configured network when updating network status' +severity = "ignore" + [[rule]] id = "benign.connect_chain" # Transport-level echo of a failure that a more specific rule reports. diff --git a/kota_tech/sd/provision.py b/kota_tech/sd/provision.py index bc59901..422a3a2 100644 --- a/kota_tech/sd/provision.py +++ b/kota_tech/sd/provision.py @@ -1,260 +1,261 @@ -"""Format an SD card for the sensorbox and write / check ENV.TXT. - -Card format required by the firmware: MBR partition table, FAT32 (its FAT -driver has no exFAT and no long file names), ENV.TXT in the root. - -Windows: formatting needs administrator rights, so kota_tech.sd.win_format -runs elevated (UAC prompt). It re-checks the disk (number, size, bus, not the -Windows drive) right before erasing it, so a card swapped after the -technician confirmed is not touched, and it writes MBR + FAT32 itself -(no Storage cmdlets, which stalled on empty card-reader slots). Larger cards -get a 32 GB FAT32 partition (Windows convention; plenty for the box). -macOS: `diskutil eraseDisk FAT32 KOTA MBRFormat` (same as format_sd.sh). -""" - -from __future__ import annotations - -import json -import os -import plistlib -import subprocess -import sys -import time -import uuid -from dataclasses import dataclass, field -from pathlib import Path - -from .disks import Disk -from .envfile import BUFFER_NAME, ENV_NAME, EnvCheck, EnvConfig, find_case_insensitive, parse_env - -VOLUME_LABEL = "KOTA" - - -@dataclass -class Outcome: - ok: bool - message: str - mount: str = "" - log: list[str] = field(default_factory=list) - - -@dataclass -class CardReport: - mount: str - filesystem: str - env: EnvCheck | None - buffered_lines: int | None # measurements waiting on the card for upload - problems: list[str] - - -# ----------------------------------------------------------------- Windows - -WORK_ROOT = Path.home() / "KotaTechnicianTool" / "tmp" - - -def _workdir() -> Path: - # Not tempfile.mkdtemp(): on Windows it creates an owner-only folder, and a - # file written there by the ELEVATED script is then unreadable for this - # (non-elevated) process. A normally inherited ACL works for both. - work = WORK_ROOT / f"sd_{uuid.uuid4().hex[:10]}" - work.mkdir(parents=True, exist_ok=False) - return work - - -def win_format_command(disk: Disk, result: Path) -> tuple[str, str, str]: - """(program, parameters, working dir) for the elevated helper.""" - root = Path(__file__).resolve().parents[2] # the version folder holding kota_tech/ - params = f'-m kota_tech.sd.win_format --disk {int(disk.id)} --size {int(disk.size)} ' \ - f'--label {VOLUME_LABEL} --result "{result}"' - return sys.executable, params, str(root) - - -def _run_elevated(program: str, params: str, cwd: str, timeout_s: int = 300) -> int | None: - """ShellExecuteEx 'runas' + wait. None if the technician refused the UAC prompt.""" - import ctypes - from ctypes import wintypes - - class SHELLEXECUTEINFOW(ctypes.Structure): - _fields_ = [("cbSize", wintypes.DWORD), ("fMask", ctypes.c_ulong), ("hwnd", wintypes.HWND), - ("lpVerb", wintypes.LPCWSTR), ("lpFile", wintypes.LPCWSTR), ("lpParameters", wintypes.LPCWSTR), - ("lpDirectory", wintypes.LPCWSTR), ("nShow", ctypes.c_int), ("hInstApp", wintypes.HINSTANCE), - ("lpIDList", ctypes.c_void_p), ("lpClass", wintypes.LPCWSTR), ("hkeyClass", wintypes.HKEY), - ("dwHotKey", wintypes.DWORD), ("hIconOrMonitor", wintypes.HANDLE), ("hProcess", wintypes.HANDLE)] - - SEE_MASK_NOCLOSEPROCESS, SW_HIDE = 0x40, 0 - info = SHELLEXECUTEINFOW(cbSize=ctypes.sizeof(SHELLEXECUTEINFOW), fMask=SEE_MASK_NOCLOSEPROCESS, - lpVerb="runas", lpFile=program, lpParameters=params, lpDirectory=cwd, nShow=SW_HIDE) - shell32 = ctypes.WinDLL("shell32", use_last_error=True) - k32 = ctypes.WinDLL("kernel32", use_last_error=True) - if not shell32.ShellExecuteExW(ctypes.byref(info)): - return None # ERROR_CANCELLED (1223) or failure - k32.WaitForSingleObject(info.hProcess, timeout_s * 1000) - code = wintypes.DWORD() - k32.GetExitCodeProcess(info.hProcess, ctypes.byref(code)) - k32.CloseHandle(info.hProcess) - return code.value - - -def _format_windows(disk: Disk, log: list[str]) -> Outcome: - result = _workdir() / "result.json" - log.append("Asking Windows for administrator rights to format the card (UAC prompt)...") - t = time.perf_counter() - code = _run_elevated(*win_format_command(disk, result)) - if code is None: - return Outcome(False, "Administrator permission was refused - the card was not formatted.", log=log) - try: - res = json.loads(result.read_text(encoding="utf-8")) - except (OSError, ValueError) as exc: - return Outcome(False, f"The formatter did not report back ({exc.__class__.__name__}, exit {code}). " - "Check the card with 'Check card'.", log=log) - steps = ", ".join(f"{k} {v:.1f} s" for k, v in res.get("timings", {}).items()) - log.append(f"{res.get('message', '')} ({time.perf_counter() - t:.1f} s incl. permission prompt; {steps})") - return Outcome(bool(res.get("ok")), res.get("message", ""), res.get("drive", ""), log) - - -def _eject_windows(mount: str) -> bool: - """Flush, lock, dismount and eject the card - what 'Safely Remove' does. - - Not Shell.Application InvokeVerb('Eject'): it fails silently because the - verb is named 'E&ject' (and is localised on non-English Windows). - """ - import ctypes - from ctypes import wintypes - - k32 = ctypes.WinDLL("kernel32", use_last_error=True) - k32.CreateFileW.restype = wintypes.HANDLE - k32.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID, - wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE] - k32.DeviceIoControl.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD, - wintypes.LPVOID, wintypes.DWORD, ctypes.POINTER(wintypes.DWORD), wintypes.LPVOID] - GENERIC_RW, SHARE_RW, OPEN_EXISTING = 0xC0000000, 0x3, 3 - FSCTL_LOCK_VOLUME, FSCTL_DISMOUNT_VOLUME = 0x00090018, 0x00090020 - IOCTL_STORAGE_MEDIA_REMOVAL, IOCTL_STORAGE_EJECT_MEDIA = 0x002D4804, 0x002D4808 - handle = k32.CreateFileW(f"\\\\.\\{mount[0]}:", GENERIC_RW, SHARE_RW, None, OPEN_EXISTING, 0, None) - if handle in (None, wintypes.HANDLE(-1).value): - return False - ret = wintypes.DWORD() - - def ioctl(code: int, buf=None, size: int = 0) -> bool: - return bool(k32.DeviceIoControl(handle, code, buf, size, None, 0, ctypes.byref(ret), None)) - - try: - k32.FlushFileBuffers(handle) - for _ in range(10): # Explorer/indexer may hold it briefly - if ioctl(FSCTL_LOCK_VOLUME): - break - time.sleep(0.5) - else: - return False - if not ioctl(FSCTL_DISMOUNT_VOLUME): - return False - allow = ctypes.c_bool(False) # PREVENT_MEDIA_REMOVAL{FALSE} - ioctl(IOCTL_STORAGE_MEDIA_REMOVAL, ctypes.byref(allow), 1) - ioctl(IOCTL_STORAGE_EJECT_MEDIA) # card readers may not support it; dismount suffices - return True - finally: - k32.CloseHandle(handle) - - -# ------------------------------------------------------------------- macOS - -def _format_macos(disk: Disk, log: list[str]) -> Outcome: - log.append(f"diskutil eraseDisk FAT32 {VOLUME_LABEL} MBRFormat {disk.id}") - proc = subprocess.run(["diskutil", "eraseDisk", "FAT32", VOLUME_LABEL, "MBRFormat", disk.id], - capture_output=True, text=True, timeout=600) - log += [l for l in proc.stdout.splitlines() if l.strip()] - if proc.returncode != 0: - return Outcome(False, "Formatting failed: " + (proc.stderr.strip() or proc.stdout.strip()[-200:]) - + " (is the card's write-protect switch on?)", log=log) - part = f"{disk.id}s1" - for _ in range(20): # wait for the volume to mount - info = plistlib.loads(subprocess.run(["diskutil", "info", "-plist", part], capture_output=True).stdout or b"") - if info.get("MountPoint"): - return Outcome(True, "Formatted FAT32 (MBR).", info["MountPoint"], log) - time.sleep(0.5) - return Outcome(False, "Formatted, but the card did not mount.", log=log) - - -def _eject_macos(disk_id: str) -> bool: - return subprocess.run(["diskutil", "eject", disk_id], capture_output=True, timeout=60).returncode == 0 - - -# ------------------------------------------------------------------ common - -def write_env(mount: str, env: EnvConfig) -> Outcome: - """Write ENV.TXT (8.3 name, LF, no BOM), keep other files (buffered data), read back.""" - problems = [i.text for i in env.problems() if i.level == "error"] - if problems: - return Outcome(False, "Settings are not valid: " + " ".join(problems)) - root = Path(mount) - if not root.is_dir(): - return Outcome(False, f"Card not found at {mount}.") - old = find_case_insensitive(root, ENV_NAME) - if old is not None and old.name != ENV_NAME: - old.unlink() # replace env.txt / Env.txt variants - target = root / ENV_NAME - data = env.render() - with open(target, "wb") as fh: - fh.write(data) - fh.flush() - os.fsync(fh.fileno()) - if target.read_bytes() != data: - return Outcome(False, "ENV.TXT read-back does not match - the card may be faulty.") - check = parse_env(target.read_bytes()) - if not check.ok: - return Outcome(False, "ENV.TXT was written but does not validate: " - + " ".join(i.text for i in check.issues if i.level == "error")) - return Outcome(True, f"ENV.TXT written and verified on {mount}.", mount) - - -def format_and_write(disk: Disk, env: EnvConfig, eject_card: bool = True) -> Outcome: - problems = [i.text for i in env.problems() if i.level == "error"] - if problems: - return Outcome(False, "Settings are not valid: " + " ".join(problems)) - log: list[str] = [] - if sys.platform == "win32": - fmt = _format_windows(disk, log) - elif sys.platform == "darwin": - fmt = _format_macos(disk, log) - else: - return Outcome(False, "SD provisioning is supported on Windows and macOS only.") - if not fmt.ok: - return fmt - wr = write_env(fmt.mount, env) - log.append(wr.message) - if not wr.ok: - return Outcome(False, wr.message, fmt.mount, log) - if eject_card: - log.append("Card ejected - you can remove it." if eject(fmt.mount, disk) else - "Could not eject the card automatically - use 'Safely remove' before pulling it out.") - return Outcome(True, "Card ready: formatted FAT32 and ENV.TXT verified.", fmt.mount, log) - - -def eject(mount: str, disk: Disk | None = None) -> bool: - """True if the card was dismounted and can be pulled out safely.""" - if sys.platform == "win32": - return _eject_windows(mount) - if sys.platform == "darwin" and disk is not None: - return _eject_macos(disk.id) - return False - - -def check_card(mount: str, filesystem: str = "") -> CardReport: - """What the box would find on this card (values masked in the UI).""" - root = Path(mount) - problems: list[str] = [] - fs = filesystem.upper() - if fs and "FAT32" not in fs and fs not in ("FAT", "MS-DOS FAT32", "MSDOS"): - problems.append(f"File system is {filesystem}; the box needs FAT32 - re-format the card.") - env_file = find_case_insensitive(root, ENV_NAME) - env = parse_env(env_file.read_bytes()) if env_file else None - if env_file is None: - problems.append("No ENV.TXT on the card - the box stops at start-up (white LED).") - elif len(env_file.name.split(".")[0]) > 8: - problems.append("ENV.TXT has a long name variant; use exactly ENV.TXT.") - buf = find_case_insensitive(root, BUFFER_NAME) - lines = None - if buf is not None: - with open(buf, "rb") as fh: - lines = sum(1 for ln in fh if ln.strip()) - return CardReport(mount, filesystem, env, lines, problems) +"""Format an SD card for the sensorbox and write / check ENV.TXT. + +Card format required by the firmware: MBR partition table, FAT32 (its FAT +driver has no exFAT and no long file names), ENV.TXT in the root. + +Windows: formatting needs administrator rights, so kota_tech.sd.win_format +runs elevated (UAC prompt). It re-checks the disk (number, size, bus, not the +Windows drive) right before erasing it, so a card swapped after the +technician confirmed is not touched, and it writes MBR + FAT32 itself +(no Storage cmdlets, which stalled on empty card-reader slots). Larger cards +get a 32 GB FAT32 partition (Windows convention; plenty for the box). +macOS: `diskutil eraseDisk FAT32 KOTA MBRFormat` (same as format_sd.sh). +""" + +from __future__ import annotations + +import json +import os +import plistlib +import subprocess +import sys +import time +import uuid +from dataclasses import dataclass, field +from pathlib import Path + +from .disks import Disk +from .envfile import BUFFER_NAME, ENV_NAME, EnvCheck, EnvConfig, find_case_insensitive, parse_env + +VOLUME_LABEL = "KOTA" + + +@dataclass +class Outcome: + ok: bool + message: str + mount: str = "" + log: list[str] = field(default_factory=list) + + +@dataclass +class CardReport: + mount: str + filesystem: str + env: EnvCheck | None + buffered_lines: int | None # measurements waiting on the card for upload + problems: list[str] + + +# ----------------------------------------------------------------- Windows + +WORK_ROOT = Path.home() / "KotaTechnicianTool" / "tmp" + + +def _workdir() -> Path: + # Not tempfile.mkdtemp(): on Windows it creates an owner-only folder, and a + # file written there by the ELEVATED script is then unreadable for this + # (non-elevated) process. A normally inherited ACL works for both. + work = WORK_ROOT / f"sd_{uuid.uuid4().hex[:10]}" + work.mkdir(parents=True, exist_ok=False) + return work + + +def win_format_command(disk: Disk, result: Path) -> tuple[str, str, str]: + """(program, parameters, working dir) for the elevated helper.""" + root = Path(__file__).resolve().parents[2] # the version folder holding kota_tech/ + params = f'-m kota_tech.sd.win_format --disk {int(disk.id)} --size {int(disk.size)} ' \ + f'--label {VOLUME_LABEL} --result "{result}"' + return sys.executable, params, str(root) + + +def _run_elevated(program: str, params: str, cwd: str, timeout_s: int = 300) -> int | None: + """ShellExecuteEx 'runas' + wait. None if the technician refused the UAC prompt.""" + import ctypes + from ctypes import wintypes + + class SHELLEXECUTEINFOW(ctypes.Structure): + _fields_ = [("cbSize", wintypes.DWORD), ("fMask", ctypes.c_ulong), ("hwnd", wintypes.HWND), + ("lpVerb", wintypes.LPCWSTR), ("lpFile", wintypes.LPCWSTR), ("lpParameters", wintypes.LPCWSTR), + ("lpDirectory", wintypes.LPCWSTR), ("nShow", ctypes.c_int), ("hInstApp", wintypes.HINSTANCE), + ("lpIDList", ctypes.c_void_p), ("lpClass", wintypes.LPCWSTR), ("hkeyClass", wintypes.HKEY), + ("dwHotKey", wintypes.DWORD), ("hIconOrMonitor", wintypes.HANDLE), ("hProcess", wintypes.HANDLE)] + + SEE_MASK_NOCLOSEPROCESS, SW_HIDE = 0x40, 0 + info = SHELLEXECUTEINFOW(cbSize=ctypes.sizeof(SHELLEXECUTEINFOW), fMask=SEE_MASK_NOCLOSEPROCESS, + lpVerb="runas", lpFile=program, lpParameters=params, lpDirectory=cwd, nShow=SW_HIDE) + shell32 = ctypes.WinDLL("shell32", use_last_error=True) + k32 = ctypes.WinDLL("kernel32", use_last_error=True) + if not shell32.ShellExecuteExW(ctypes.byref(info)): + return None # ERROR_CANCELLED (1223) or failure + k32.WaitForSingleObject(info.hProcess, timeout_s * 1000) + code = wintypes.DWORD() + k32.GetExitCodeProcess(info.hProcess, ctypes.byref(code)) + k32.CloseHandle(info.hProcess) + return code.value + + +def _format_windows(disk: Disk, log: list[str]) -> Outcome: + result = _workdir() / "result.json" + log.append("Asking Windows for administrator rights to format the card (UAC prompt)...") + t = time.perf_counter() + code = _run_elevated(*win_format_command(disk, result)) + if code is None: + return Outcome(False, "Administrator permission was refused - the card was not formatted.", log=log) + try: + res = json.loads(result.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + return Outcome(False, f"The formatter did not report back ({exc.__class__.__name__}, exit {code}). " + "Check the card with 'Check card'.", log=log) + steps = ", ".join(f"{k} {v:.1f} s" for k, v in res.get("timings", {}).items()) + log.append(f"{res.get('message', '')} ({time.perf_counter() - t:.1f} s incl. permission prompt; {steps})") + return Outcome(bool(res.get("ok")), res.get("message", ""), res.get("drive", ""), log) + + +def _eject_windows(mount: str) -> bool: + """Flush, lock, dismount and eject the card - what 'Safely Remove' does. + + Not Shell.Application InvokeVerb('Eject'): it fails silently because the + verb is named 'E&ject' (and is localised on non-English Windows). + """ + import ctypes + from ctypes import wintypes + + k32 = ctypes.WinDLL("kernel32", use_last_error=True) + k32.CreateFileW.restype = wintypes.HANDLE + k32.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID, + wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE] + k32.DeviceIoControl.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD, + wintypes.LPVOID, wintypes.DWORD, ctypes.POINTER(wintypes.DWORD), wintypes.LPVOID] + GENERIC_RW, SHARE_RW, OPEN_EXISTING = 0xC0000000, 0x3, 3 + FSCTL_LOCK_VOLUME, FSCTL_DISMOUNT_VOLUME = 0x00090018, 0x00090020 + IOCTL_STORAGE_MEDIA_REMOVAL, IOCTL_STORAGE_EJECT_MEDIA = 0x002D4804, 0x002D4808 + handle = k32.CreateFileW(f"\\\\.\\{mount[0]}:", GENERIC_RW, SHARE_RW, None, OPEN_EXISTING, 0, None) + if handle in (None, wintypes.HANDLE(-1).value): + return False + ret = wintypes.DWORD() + + def ioctl(code: int, buf=None, size: int = 0) -> bool: + return bool(k32.DeviceIoControl(handle, code, buf, size, None, 0, ctypes.byref(ret), None)) + + try: + k32.FlushFileBuffers(handle) + for _ in range(10): # Explorer/indexer may hold it briefly + if ioctl(FSCTL_LOCK_VOLUME): + break + time.sleep(0.5) + else: + return False + if not ioctl(FSCTL_DISMOUNT_VOLUME): + return False + allow = ctypes.c_bool(False) # PREVENT_MEDIA_REMOVAL{FALSE} + ioctl(IOCTL_STORAGE_MEDIA_REMOVAL, ctypes.byref(allow), 1) + ioctl(IOCTL_STORAGE_EJECT_MEDIA) # card readers may not support it; dismount suffices + return True + finally: + k32.CloseHandle(handle) + + +# ------------------------------------------------------------------- macOS + +def _format_macos(disk: Disk, log: list[str]) -> Outcome: + log.append(f"diskutil eraseDisk FAT32 {VOLUME_LABEL} MBRFormat {disk.id}") + proc = subprocess.run(["diskutil", "eraseDisk", "FAT32", VOLUME_LABEL, "MBRFormat", disk.id], + capture_output=True, text=True, timeout=600) + log += [l for l in proc.stdout.splitlines() if l.strip()] + if proc.returncode != 0: + return Outcome(False, "Formatting failed: " + (proc.stderr.strip() or proc.stdout.strip()[-200:]) + + " (is the card's write-protect switch on?)", log=log) + part = f"{disk.id}s1" + for _ in range(20): # wait for the volume to mount + info = plistlib.loads(subprocess.run(["diskutil", "info", "-plist", part], capture_output=True).stdout or b"") + if info.get("MountPoint"): + return Outcome(True, "Formatted FAT32 (MBR).", info["MountPoint"], log) + time.sleep(0.5) + return Outcome(False, "Formatted, but the card did not mount.", log=log) + + +def _eject_macos(disk_id: str) -> bool: + return subprocess.run(["diskutil", "eject", disk_id], capture_output=True, timeout=60).returncode == 0 + + +# ------------------------------------------------------------------ common + +def write_env(mount: str, env: EnvConfig) -> Outcome: + """Write ENV.TXT (8.3 name, LF, no BOM), keep other files (buffered data), read back.""" + problems = [i.text for i in env.problems() if i.level == "error"] + if problems: + return Outcome(False, "Settings are not valid: " + " ".join(problems)) + root = Path(mount) + if not root.is_dir(): + return Outcome(False, f"Card not found at {mount}.") + old = find_case_insensitive(root, ENV_NAME) + if old is not None and old.name != ENV_NAME: + old.unlink() # replace env.txt / Env.txt variants + target = root / ENV_NAME + data = env.render() + with open(target, "wb") as fh: + fh.write(data) + fh.flush() + os.fsync(fh.fileno()) + if target.read_bytes() != data: + return Outcome(False, "ENV.TXT read-back does not match - the card may be faulty.") + check = parse_env(target.read_bytes()) + if not check.ok: + return Outcome(False, "ENV.TXT was written but does not validate: " + + " ".join(i.text for i in check.issues if i.level == "error")) + return Outcome(True, f"ENV.TXT written and verified on {mount}.", mount) + + +def format_and_write(disk: Disk, env: EnvConfig, eject_card: bool = True) -> Outcome: + problems = [i.text for i in env.problems() if i.level == "error"] + if problems: + return Outcome(False, "Settings are not valid: " + " ".join(problems)) + log: list[str] = [] + if sys.platform == "win32": + fmt = _format_windows(disk, log) + elif sys.platform == "darwin": + fmt = _format_macos(disk, log) + else: + return Outcome(False, "SD provisioning is supported on Windows and macOS only.") + if not fmt.ok: + return fmt + wr = write_env(fmt.mount, env) + log.append(wr.message) + if not wr.ok: + return Outcome(False, wr.message, fmt.mount, log) + if eject_card: + log.append("Card ejected - you can remove it." if eject(fmt.mount, disk) else + "Could not eject the card automatically - use 'Safely remove' before pulling it out.") + return Outcome(True, "Card ready: formatted FAT32 and ENV.TXT verified.", fmt.mount, log) + + +def eject(mount: str, disk: Disk | None = None) -> bool: + """True if the card was dismounted and can be pulled out safely.""" + if sys.platform == "win32": + return _eject_windows(mount) + if sys.platform == "darwin" and disk is not None: + return _eject_macos(disk.id) + return False + + +def check_card(mount: str, filesystem: str = "") -> CardReport: + """What the box would find on this card (values masked in the UI).""" + root = Path(mount) + problems: list[str] = [] + # Windows says "FAT32"; macOS diskutil says "DOS_FAT_32" / "Windows_FAT_32" / "MS-DOS FAT32". + fs = filesystem.upper().replace("_", "").replace(" ", "") + if fs and "FAT32" not in fs and fs not in ("FAT", "MSDOS"): + problems.append(f"File system is {filesystem}; the box needs FAT32 - re-format the card.") + env_file = find_case_insensitive(root, ENV_NAME) + env = parse_env(env_file.read_bytes()) if env_file else None + if env_file is None: + problems.append("No ENV.TXT on the card - the box stops at start-up (white LED).") + elif len(env_file.name.split(".")[0]) > 8: + problems.append("ENV.TXT has a long name variant; use exactly ENV.TXT.") + buf = find_case_insensitive(root, BUFFER_NAME) + lines = None + if buf is not None: + with open(buf, "rb") as fh: + lines = sum(1 for ln in fh if ln.strip()) + return CardReport(mount, filesystem, env, lines, problems) diff --git a/pyproject.toml b/pyproject.toml index f8623e3..f8ec057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "kota-technician-tool" -version = "0.7.0" +version = "0.8.0" description = "KOTA sensorbox technician diagnostic tool (demo)" requires-python = ">=3.11,<3.14" dependencies = [ diff --git a/run_macos.command b/run_macos.command index 361a151..0a15bf5 100755 --- a/run_macos.command +++ b/run_macos.command @@ -2,8 +2,10 @@ # Double-click in Finder, or run: bash run_macos.command cd "$(dirname "$0")" || exit 1 +# If Finder refuses to open this file (Google Drive drops the executable bit), run it once as: +# bash run_macos.command (or: chmod +x run_macos.command) # Keep the virtual environment out of the (Google Drive) tool folder. -export UV_PROJECT_ENVIRONMENT="$HOME/Library/Caches/KotaTechnicianTool/venv-0.7" +export UV_PROJECT_ENVIRONMENT="$HOME/Library/Caches/KotaTechnicianTool/venv-0.8" export PATH="$HOME/.local/bin:$PATH" if ! command -v uv >/dev/null 2>&1; then diff --git a/run_windows.bat b/run_windows.bat index f000e51..180a59e 100644 --- a/run_windows.bat +++ b/run_windows.bat @@ -3,7 +3,7 @@ setlocal cd /d "%~dp0" rem Keep the virtual environment out of the (Google Drive) tool folder. -set "UV_PROJECT_ENVIRONMENT=%LOCALAPPDATA%\KotaTechnicianTool\venv-0.7" +set "UV_PROJECT_ENVIRONMENT=%LOCALAPPDATA%\KotaTechnicianTool\venv-0.8" set "PATH=%USERPROFILE%\.local\bin;%PATH%" where uv >nul 2>nul diff --git a/tests/test_engine.py b/tests/test_engine.py index d447bfd..2c37fff 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -247,3 +247,13 @@ def test_live_brownout_loop_is_a_power_problem(): assert top.finding.rule_id == "reboot.brownout" and top.finding.count >= 5 assert "reboot.loop" in {c.rule_id for c in top.consequences} assert r.verdict.startswith("Problem: Power supply") + + +@pytest.mark.parametrize("line", [ + "E (2893) wifi:sta is connecting, return error", + "E (2892) chip[DL]: esp_wifi_connect() failed: ESP_ERR_WIFI_CONN", + "E (5120) chip[DL]: Failed to get configured network when updating network status: Error ESP32:0x0500300F", +]) +def test_harmless_wifi_start_errors_are_not_reported(line): + _, r = run(BOOT + APP + [line]) + assert r.roots == [] and "Unrecognised" not in r.to_text() diff --git a/tests/test_ports.py b/tests/test_ports.py index d810c42..45c2123 100644 --- a/tests/test_ports.py +++ b/tests/test_ports.py @@ -12,3 +12,15 @@ def test_other_ports_not_sensorbox(): bt = PortInfo("COM3", "Standard Serial over Bluetooth link", None, None, None) assert not bt.is_sensorbox and bt.mac is None assert bt.label == "COM3 - Standard Serial over Bluetooth link" + + +def test_macos_lists_only_callout_devices(monkeypatch): + from types import SimpleNamespace + + from kota_tech.serialio import ports + fake = [SimpleNamespace(device=d, description="USB JTAG/serial debug unit", vid=0x303A, pid=0x1001, + serial_number="A4:CB:8F:C9:46:F8") + for d in ("/dev/tty.usbmodem1101", "/dev/cu.usbmodem1101")] + monkeypatch.setattr(ports.sys, "platform", "darwin") + monkeypatch.setattr(ports.list_ports, "comports", lambda: fake) + assert [p.device for p in ports.list_serial_ports()] == ["/dev/cu.usbmodem1101"] diff --git a/tests/test_sd.py b/tests/test_sd.py index 4e26e6a..68551b3 100644 --- a/tests/test_sd.py +++ b/tests/test_sd.py @@ -170,3 +170,14 @@ def test_elevated_result_folder_is_not_owner_only(tmp_path, monkeypatch): assert work.parent == tmp_path / "tmp" and work.is_dir() (work / "result.json").write_text('{"ok": true}', encoding="utf-8") assert (work / "result.json").read_text(encoding="utf-8") + + +@pytest.mark.parametrize("fs", ["FAT32", "DOS_FAT_32", "Windows_FAT_32", "MS-DOS FAT32"]) +def test_fat32_names_from_windows_and_macos_are_accepted(tmp_path, fs): + (tmp_path / "ENV.TXT").write_bytes(b"INFLUX_URL=https://x\nINFLUX_TOKEN=abc\n") + assert not any("File system" in p for p in check_card(str(tmp_path), fs).problems) + + +def test_exfat_is_flagged(tmp_path): + (tmp_path / "ENV.TXT").write_bytes(b"INFLUX_URL=https://x\nINFLUX_TOKEN=abc\n") + assert any("File system" in p for p in check_card(str(tmp_path), "ExFAT").problems) diff --git a/uv.lock b/uv.lock index 491d840..f56f5b6 100644 --- a/uv.lock +++ b/uv.lock @@ -250,7 +250,7 @@ wheels = [ [[package]] name = "kota-technician-tool" -version = "0.7.0" +version = "0.8.0" source = { virtual = "." } dependencies = [ { name = "esptool" },