#!/usr/bin/env python3
"""NW AL Scanner Radio Intelligence - Incident Intelligence Phase 1.

Monitor-only service. Watches configured recording folders, waits for completed
files, identifies source/channel metadata, archives audio, transcribes locally
with whisper.cpp when installed, and marks likely duplicate Lauderdale paging
repeater copies, provides operator audio playback and transcript correction,
and stores correction vocabulary. It does NOT publish, create incidents, or modify
Incident Command in Phase 3.
"""
from __future__ import annotations

import difflib
import hashlib
import json
import os
import re
import shutil
import sqlite3
import subprocess
import threading
import time
import mimetypes
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import urlparse

BASE_DIR = Path(__file__).resolve().parent
TENANT_ID = (os.environ.get('NWALS_TENANT') or 'nw-al-scanner').strip()
PORT = int(os.environ.get('NWALS_RADIO_PORT') or '9021')
CONFIG_PATH = Path(os.environ.get('NWALS_RADIO_CONFIG') or (BASE_DIR / 'tenants' / TENANT_ID / 'radio_intelligence.json'))

LOCALAPPDATA = Path(os.environ.get('LOCALAPPDATA') or (Path.home() / '.local' / 'share'))
DATA_ROOT = Path(os.environ.get('NWALS_RADIO_DATA_DIR') or (LOCALAPPDATA / 'NWALScannerPlatform' / 'data' / 'radio_intelligence' / TENANT_ID))
ARCHIVE_ROOT = DATA_ROOT / 'audio'
TRANSCRIPT_ROOT = DATA_ROOT / 'transcripts'
DB_PATH = DATA_ROOT / 'radio_intelligence.sqlite3'
DATA_ROOT.mkdir(parents=True, exist_ok=True)
ARCHIVE_ROOT.mkdir(parents=True, exist_ok=True)
TRANSCRIPT_ROOT.mkdir(parents=True, exist_ok=True)

TOOLS_ROOT = BASE_DIR / 'tools' / 'radio_transcription'
DEFAULT_WHISPER_EXE = TOOLS_ROOT / 'whisper' / 'whisper-cli.exe'
DEFAULT_WHISPER_MODEL = TOOLS_ROOT / 'models' / 'ggml-base.en.bin'
DEFAULT_FW_PYTHON = TOOLS_ROOT / 'faster_whisper' / 'venv' / 'Scripts' / 'python.exe'
DEFAULT_FW_HELPER = BASE_DIR / 'faster_whisper_transcribe.py'

LOCK = threading.RLock()
STOP = threading.Event()
STARTED = time.time()
STABILITY_SECONDS = 1.5
POLL_SECONDS = 1.0
TRANSCRIBE_POLL_SECONDS = 1.5
DEEPGRAM_POLL_SECONDS = 1.2
DEEPGRAM_KEY_PATH = DATA_ROOT / 'deepgram_api_key.txt'

DEFAULT_CONFIG = {
    'enabled': True,
    'monitor_only': True,
    'ingest_existing_on_first_run': False,
    'archive_audio': True,
    'transcription': {
        'enabled': True,
        'backend': 'faster_whisper',
        'python': str(DEFAULT_FW_PYTHON),
        'helper': str(DEFAULT_FW_HELPER),
        'model': 'distil-large-v3',
        'compute_type': 'int8_float16',
        'device': 'cuda',
        'beam_size': 5,
        'patience': 1.5,
        'cpu_threads': 4,
        'fallback_model': 'small.en',
        'fallback_device': 'cpu',
        'fallback_compute_type': 'int8',
        'vad_filter': True,
        'language': 'en',
        'initial_prompt': 'Public safety radio traffic in northwest Alabama. Fire, EMS, dispatch, street names, road names, unit numbers and local place names.'
    },
    'deepgram': {
        'enabled': False,
        'model': 'nova-2-phonecall',
        'language': 'en',
        'smart_format': True,
        'punctuate': True,
        'mip_opt_out': True,
        'keyword_boost': 2.0,
        'static_keywords': ['wreck', 'injuries', 'Lauderdale', 'Florence', 'Colbert', 'Killen', 'Tuscumbia', 'Sheffield', 'Muscle', 'Shoals', 'Helton', 'Mars', 'Hill']
    },
    'incident_grouping': {
        'enabled': True,
        'lookback_hours': 12,
        'max_gap_seconds': 1200,
        'cross_source_fast_window_seconds': 240,
        'same_type_window_seconds': 900,
        'min_words': 4
    },
    'duplicate_detection': {
        'enabled': True,
        'watcher_ids': ['vox'],
        'window_seconds': 90,
        'similarity_threshold': 0.78,
        'min_words': 4,
        'group_label': 'Lauderdale Paging Repeat'
    },
    'watchers': [
        {
            'id': 'sdrtrunk',
            'name': 'SDRTrunk Calls',
            'adapter': 'sdrtrunk',
            'enabled': True,
            'path': r'%USERPROFILE%\\SDRTrunk\\recordings',
            'extensions': ['.mp3', '.wav']
        },
        {
            'id': 'vox',
            'name': 'Lauderdale VOX Recordings',
            'adapter': 'vox',
            'enabled': True,
            'path': r'C:\\VOX Recorder\\Recordings',
            'source_name': 'Lauderdale County VOX',
            'extensions': ['.mp3', '.wav']
        }
    ]
}


def deep_defaults(dst: dict, src: dict) -> dict:
    for key, val in src.items():
        if key not in dst:
            dst[key] = val
        elif isinstance(val, dict) and isinstance(dst.get(key), dict):
            deep_defaults(dst[key], val)
    return dst


def load_config() -> dict:
    if not CONFIG_PATH.exists():
        CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
        CONFIG_PATH.write_text(json.dumps(DEFAULT_CONFIG, indent=2) + '\n', encoding='utf-8')
    try:
        cfg = json.loads(CONFIG_PATH.read_text(encoding='utf-8-sig'))
    except Exception:
        cfg = json.loads(json.dumps(DEFAULT_CONFIG))
    deep_defaults(cfg, DEFAULT_CONFIG)
    return cfg


def save_config(cfg: dict) -> None:
    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
    tmp = CONFIG_PATH.with_suffix('.json.tmp')
    tmp.write_text(json.dumps(cfg, indent=2) + '\n', encoding='utf-8')
    tmp.replace(CONFIG_PATH)


def expand_path(text: str) -> Path:
    return Path(os.path.expandvars(os.path.expanduser(text))).resolve()


def _ensure_column(con: sqlite3.Connection, table: str, name: str, ddl: str) -> None:
    cols = {r[1] for r in con.execute(f'PRAGMA table_info({table})').fetchall()}
    if name not in cols:
        con.execute(f'ALTER TABLE {table} ADD COLUMN {name} {ddl}')


def db() -> sqlite3.Connection:
    con = sqlite3.connect(DB_PATH, timeout=10)
    con.row_factory = sqlite3.Row
    con.execute('PRAGMA journal_mode=WAL')
    con.execute('''CREATE TABLE IF NOT EXISTS files_seen (
        fingerprint TEXT PRIMARY KEY,
        original_path TEXT NOT NULL,
        size_bytes INTEGER NOT NULL,
        mtime REAL NOT NULL,
        first_seen REAL NOT NULL,
        disposition TEXT NOT NULL
    )''')
    con.execute('''CREATE TABLE IF NOT EXISTS transmissions (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        fingerprint TEXT UNIQUE NOT NULL,
        timestamp_epoch REAL NOT NULL,
        timestamp_text TEXT NOT NULL,
        adapter TEXT NOT NULL,
        watcher_id TEXT NOT NULL,
        watcher_name TEXT NOT NULL,
        source_name TEXT NOT NULL,
        system_name TEXT,
        site_name TEXT,
        alias_name TEXT,
        target_id TEXT,
        original_path TEXT NOT NULL,
        archive_path TEXT,
        filename TEXT NOT NULL,
        extension TEXT NOT NULL,
        size_bytes INTEGER NOT NULL,
        ingested_at REAL NOT NULL,
        status TEXT NOT NULL DEFAULT 'MONITOR_ONLY'
    )''')
    _ensure_column(con, 'transmissions', 'transcript', 'TEXT')
    _ensure_column(con, 'transmissions', 'transcript_status', "TEXT DEFAULT 'PENDING'")
    _ensure_column(con, 'transmissions', 'transcript_error', 'TEXT')
    _ensure_column(con, 'transmissions', 'transcribed_at', 'REAL')
    _ensure_column(con, 'transmissions', 'transcript_backend', 'TEXT')
    _ensure_column(con, 'transmissions', 'duplicate_group', 'TEXT')
    _ensure_column(con, 'transmissions', 'duplicate_of_id', 'INTEGER')
    _ensure_column(con, 'transmissions', 'duplicate_similarity', 'REAL')
    _ensure_column(con, 'transmissions', 'corrected_transcript', 'TEXT')
    _ensure_column(con, 'transmissions', 'corrected_at', 'REAL')
    _ensure_column(con, 'transmissions', 'corrected_by', 'TEXT')
    _ensure_column(con, 'transmissions', 'deepgram_transcript', 'TEXT')
    _ensure_column(con, 'transmissions', 'deepgram_status', "TEXT DEFAULT 'PENDING'")
    _ensure_column(con, 'transmissions', 'deepgram_error', 'TEXT')
    _ensure_column(con, 'transmissions', 'deepgram_transcribed_at', 'REAL')
    _ensure_column(con, 'transmissions', 'deepgram_model', 'TEXT')
    _ensure_column(con, 'transmissions', 'deepgram_confidence', 'REAL')
    con.execute('''CREATE TABLE IF NOT EXISTS transcript_corrections (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        transmission_id INTEGER NOT NULL,
        raw_transcript TEXT,
        corrected_transcript TEXT NOT NULL,
        corrected_at REAL NOT NULL,
        corrected_by TEXT NOT NULL DEFAULT 'operator'
    )''')
    con.execute('''CREATE TABLE IF NOT EXISTS vocabulary_phrases (
        phrase TEXT PRIMARY KEY,
        first_seen REAL NOT NULL,
        last_seen REAL NOT NULL,
        use_count INTEGER NOT NULL DEFAULT 1,
        source TEXT NOT NULL DEFAULT 'operator_correction'
    )''')
    con.execute('CREATE TABLE IF NOT EXISTS incident_handoffs (candidate_id TEXT PRIMARY KEY, sent_at REAL NOT NULL, incident_url TEXT NOT NULL, payload_json TEXT NOT NULL)')
    con.execute('CREATE INDEX IF NOT EXISTS idx_transmissions_transcript_status ON transmissions(transcript_status, id)')
    con.execute('CREATE INDEX IF NOT EXISTS idx_transmissions_watcher_time ON transmissions(watcher_id, timestamp_epoch)')
    con.execute('CREATE INDEX IF NOT EXISTS idx_transmissions_deepgram_status ON transmissions(deepgram_status, id)')
    con.commit()
    return con


def fingerprint(path: Path, stat: os.stat_result) -> str:
    raw = f'{path.resolve()}|{stat.st_size}|{stat.st_mtime_ns}'.encode('utf-8', errors='replace')
    return hashlib.sha1(raw).hexdigest()


SDR_RE = re.compile(r'^(?P<date>\d{8})_(?P<time>\d{6})(?P<body>.+?)(?:__TO_(?P<target>[^.]+))?$', re.I)


def parse_sdrtrunk(path: Path, st: os.stat_result) -> dict:
    stem = path.stem
    m = SDR_RE.match(stem)
    epoch = st.st_mtime
    system = site = alias = source = 'SDRTrunk'
    target = ''
    if m:
        try:
            dt = datetime.strptime(m.group('date') + m.group('time'), '%Y%m%d%H%M%S')
            epoch = dt.timestamp()
        except Exception:
            pass
        body = m.group('body').strip('_')
        target = (m.group('target') or '').strip('_')
        human = body.replace('_', ' ')
        known = [
            'Florence Fire', 'Colbert Fire', 'Muscle Shoals Fire', 'Sheffield Fire',
            'Lauderdale EMA North', 'Tuscumbia Fire'
        ]
        alias = next((k for k in known if human.lower().endswith(k.lower())), '')
        source = alias or human
        if alias:
            prefix = human[:-len(alias)].strip()
            system = prefix or alias
            site = prefix or alias
        else:
            alias = source
    return {
        'timestamp_epoch': epoch,
        'source_name': source,
        'system_name': system,
        'site_name': site,
        'alias_name': alias,
        'target_id': target,
    }


def parse_generic(path: Path, st: os.stat_result, watcher: dict) -> dict:
    return {
        'timestamp_epoch': st.st_mtime,
        'source_name': watcher.get('source_name') or watcher.get('name') or 'Radio Audio',
        'system_name': watcher.get('system_name') or '',
        'site_name': watcher.get('site_name') or '',
        'alias_name': watcher.get('source_name') or watcher.get('name') or '',
        'target_id': '',
    }


def safe_slug(text: str) -> str:
    s = re.sub(r'[^A-Za-z0-9._-]+', '_', text.strip())
    return s.strip('_')[:80] or 'unknown'


def archive_file(path: Path, meta: dict, cfg: dict) -> str:
    if not cfg.get('archive_audio', True):
        return ''
    dt = datetime.fromtimestamp(meta['timestamp_epoch'])
    outdir = ARCHIVE_ROOT / dt.strftime('%Y-%m-%d') / safe_slug(meta['source_name'])
    outdir.mkdir(parents=True, exist_ok=True)
    dest = outdir / path.name
    if dest.exists() and dest.stat().st_size == path.stat().st_size:
        return str(dest)
    if dest.exists():
        dest = outdir / f'{path.stem}_{int(time.time()*1000)}{path.suffix}'
    shutil.copy2(path, dest)
    return str(dest)


class IngestEngine:
    def __init__(self):
        self.pending: dict[str, tuple[int, float]] = {}
        self.last_scan = 0.0
        self.last_ingest = 0.0
        self.last_error = ''
        self.count_session = 0

    def already_seen(self, fp: str) -> bool:
        con = db()
        try:
            return con.execute('SELECT 1 FROM files_seen WHERE fingerprint=?', (fp,)).fetchone() is not None
        finally:
            con.close()

    def mark_seen(self, fp: str, path: Path, st: os.stat_result, disposition: str):
        con = db()
        try:
            con.execute('INSERT OR IGNORE INTO files_seen VALUES (?,?,?,?,?,?)',
                        (fp, str(path), st.st_size, st.st_mtime, time.time(), disposition))
            con.commit()
        finally:
            con.close()

    def insert_transmission(self, fp: str, path: Path, st: os.stat_result, watcher: dict, meta: dict, archive_path: str):
        ts = datetime.fromtimestamp(meta['timestamp_epoch'])
        con = db()
        try:
            con.execute('''INSERT OR IGNORE INTO transmissions
                (fingerprint,timestamp_epoch,timestamp_text,adapter,watcher_id,watcher_name,source_name,
                 system_name,site_name,alias_name,target_id,original_path,archive_path,filename,extension,
                 size_bytes,ingested_at,status,transcript_status)
                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', (
                fp, meta['timestamp_epoch'], ts.strftime('%Y-%m-%d %H:%M:%S'), watcher.get('adapter','generic'),
                watcher.get('id','watcher'), watcher.get('name','Watcher'), meta['source_name'], meta.get('system_name',''),
                meta.get('site_name',''), meta.get('alias_name',''), meta.get('target_id',''), str(path), archive_path,
                path.name, path.suffix.lower(), st.st_size, time.time(), 'MONITOR_ONLY', 'PENDING'))
            con.commit()
        finally:
            con.close()

    def scan_once(self):
        cfg = load_config()
        if not cfg.get('enabled', True):
            return
        now = time.time()
        self.last_scan = now
        for watcher in cfg.get('watchers', []):
            if not watcher.get('enabled', True):
                continue
            folder = expand_path(str(watcher.get('path') or ''))
            if not folder.exists() or not folder.is_dir():
                continue
            exts = {str(x).lower() for x in watcher.get('extensions', ['.mp3', '.wav'])}
            try:
                items = list(folder.iterdir())
            except Exception as exc:
                self.last_error = f'{watcher.get("name","watcher")}: {exc}'
                continue
            for path in items:
                try:
                    if not path.is_file() or path.suffix.lower() not in exts:
                        continue
                    st = path.stat()
                    fp = fingerprint(path, st)
                    if self.already_seen(fp):
                        self.pending.pop(str(path), None)
                        continue
                    if not cfg.get('ingest_existing_on_first_run', False) and st.st_mtime < STARTED - 3:
                        self.mark_seen(fp, path, st, 'BASELINE_SKIPPED')
                        continue
                    key = str(path)
                    prior = self.pending.get(key)
                    if not prior or prior[0] != st.st_size:
                        self.pending[key] = (st.st_size, now)
                        continue
                    if now - prior[1] < STABILITY_SECONDS:
                        continue
                    adapter = str(watcher.get('adapter') or 'generic').lower()
                    meta = parse_sdrtrunk(path, st) if adapter == 'sdrtrunk' else parse_generic(path, st, watcher)
                    archived = archive_file(path, meta, cfg)
                    self.insert_transmission(fp, path, st, watcher, meta, archived)
                    self.mark_seen(fp, path, st, 'INGESTED')
                    self.pending.pop(key, None)
                    self.last_ingest = now
                    self.count_session += 1
                except Exception as exc:
                    self.last_error = f'{path.name}: {exc}'

    def loop(self):
        while not STOP.wait(POLL_SECONDS):
            try:
                self.scan_once()
            except Exception as exc:
                self.last_error = str(exc)


class TranscriptionEngine:
    def __init__(self):
        self.last_attempt = 0.0
        self.last_success = 0.0
        self.last_error = ''
        self.count_session = 0
        self.current_id = None

    def backend_status(self) -> dict:
        cfg = load_config().get('transcription', {})
        backend = str(cfg.get('backend') or 'faster_whisper')
        if backend == 'faster_whisper':
            py = expand_path(str((BASE_DIR / cfg.get('python')).resolve()) if cfg.get('python') and not Path(str(cfg.get('python'))).is_absolute() else str(cfg.get('python') or DEFAULT_FW_PYTHON))
            helper = expand_path(str((BASE_DIR / cfg.get('helper')).resolve()) if cfg.get('helper') and not Path(str(cfg.get('helper'))).is_absolute() else str(cfg.get('helper') or DEFAULT_FW_HELPER))
            ready = bool(cfg.get('enabled', True)) and py.exists() and helper.exists()
            return {'enabled': bool(cfg.get('enabled', True)), 'backend': backend, 'python': str(py), 'helper': str(helper), 'model': str(cfg.get('model') or 'distil-large-v3'), 'ready': ready, 'setup_script': str(BASE_DIR / 'SETUP_RADIO_TRANSCRIPTION.bat')}
        exe = expand_path(str((BASE_DIR / cfg.get('executable')).resolve()) if cfg.get('executable') and not Path(str(cfg.get('executable'))).is_absolute() else str(cfg.get('executable') or DEFAULT_WHISPER_EXE))
        model = expand_path(str((BASE_DIR / cfg.get('model')).resolve()) if cfg.get('model') and not Path(str(cfg.get('model'))).is_absolute() else str(cfg.get('model') or DEFAULT_WHISPER_MODEL))
        ready = bool(cfg.get('enabled', True)) and exe.exists() and model.exists()
        return {'enabled': bool(cfg.get('enabled', True)), 'backend': backend, 'executable': str(exe), 'model': str(model), 'ready': ready, 'setup_script': str(BASE_DIR / 'SETUP_RADIO_TRANSCRIPTION.bat')}

    def _next_pending(self):
        con = db()
        try:
            # Include V066 records that predate the transcript columns/migration.
            row = con.execute('''SELECT * FROM transmissions
                WHERE COALESCE(transcript_status,'PENDING') IN ('PENDING','WAITING_SETUP','RETRY')
                ORDER BY timestamp_epoch ASC, id ASC LIMIT 1''').fetchone()
            return dict(row) if row else None
        finally:
            con.close()

    def _set_status(self, txid: int, status: str, error: str = ''):
        con = db()
        try:
            con.execute('UPDATE transmissions SET transcript_status=?, transcript_error=? WHERE id=?', (status, error, txid))
            con.commit()
        finally:
            con.close()

    def _audio_path(self, row: dict) -> Path | None:
        for candidate in (row.get('archive_path'), row.get('original_path')):
            if candidate:
                p = Path(candidate)
                if p.exists() and p.is_file():
                    return p
        return None

    def transcribe_one(self, row: dict):
        status = self.backend_status()
        if not status['enabled']:
            self._set_status(row['id'], 'DISABLED')
            return
        if not status['ready']:
            self._set_status(row['id'], 'WAITING_SETUP', 'Run SETUP_RADIO_TRANSCRIPTION.bat to install local whisper.cpp and the base.en model.')
            time.sleep(3)
            return
        audio = self._audio_path(row)
        if not audio:
            self._set_status(row['id'], 'AUDIO_MISSING', 'Neither archived nor original audio file is available.')
            return
        cfg = load_config().get('transcription', {})
        # V087 stabilization: use only the static geographic/public-safety prompt here.
        # Operator corrections remain stored in vocabulary_phrases for future use, but feeding
        # full corrected transmissions back into Whisper can seed unrelated hallucinations.
        prompt = str(cfg.get('initial_prompt') or '').strip()
        self.current_id = row['id']; self.last_attempt = time.time(); self._set_status(row['id'], 'TRANSCRIBING')
        try:
            backend = str(cfg.get('backend') or 'faster_whisper')
            if backend == 'faster_whisper':
                cmd = [status['python'], status['helper'], '--audio', str(audio), '--model', str(cfg.get('model') or 'distil-large-v3'), '--language', str(cfg.get('language') or 'en'), '--compute-type', str(cfg.get('compute_type') or 'int8_float16'), '--device', str(cfg.get('device') or 'cuda'), '--beam-size', str(int(cfg.get('beam_size') or 5)), '--patience', str(float(cfg.get('patience') or 1.5)), '--cpu-threads', str(int(cfg.get('cpu_threads') or 4)), '--fallback-model', str(cfg.get('fallback_model') or 'small.en'), '--fallback-device', str(cfg.get('fallback_device') or 'cpu'), '--fallback-compute-type', str(cfg.get('fallback_compute_type') or 'int8')]
                if prompt: cmd += ['--prompt', prompt[:1800]]
                if bool(cfg.get('vad_filter', True)): cmd += ['--vad-filter']
                proc = subprocess.run(cmd, capture_output=True, text=True, timeout=300, encoding='utf-8', errors='replace', creationflags=(0x08000000 if os.name == 'nt' else 0))
                if proc.returncode != 0:
                    msg=(proc.stderr or proc.stdout or f'faster-whisper exit {proc.returncode}').strip()[-1600:]; self._set_status(row['id'],'RETRY',msg); self.last_error=msg; return
                payload=json.loads((proc.stdout or '{}').strip().splitlines()[-1]); text=re.sub(r'\s+',' ',str(payload.get('text') or '')).strip(); backend_label=f"faster-whisper/{payload.get('actual_model') or cfg.get('model') or 'distil-large-v3'}@{payload.get('actual_device') or cfg.get('device') or 'cuda'}" + ('/fallback' if payload.get('fallback_used') else '')
            else:
                exe=Path(status['executable']); model=Path(status['model']); outbase=TRANSCRIPT_ROOT / f'tx_{row["id"]}_{safe_slug(row.get("source_name") or "radio")}'
                txt_path=Path(str(outbase)+'.txt')
                try:
                    if txt_path.exists(): txt_path.unlink()
                except Exception: pass
                cmd=[str(exe),'-m',str(model),'-f',str(audio),'-l',str(cfg.get('language') or 'en'),'-otxt','-of',str(outbase),'-nt','-np']
                if prompt: cmd += ['--prompt',prompt]
                proc=subprocess.run(cmd,capture_output=True,text=True,timeout=180,encoding='utf-8',errors='replace',creationflags=(0x08000000 if os.name=='nt' else 0))
                if proc.returncode != 0:
                    msg=(proc.stderr or proc.stdout or f'whisper-cli exit {proc.returncode}').strip()[-1200:]; self._set_status(row['id'],'RETRY',msg); self.last_error=msg; return
                text=txt_path.read_text(encoding='utf-8-sig',errors='replace').strip() if txt_path.exists() else (proc.stdout or '').strip(); text=re.sub(r'\s+',' ',text).strip(); backend_label='whisper.cpp/base.en'
            if text.lower() in {'[music]','[applause]','thank you.','thanks for watching.','you'}: text=''
            text=re.sub(r'(?i)^(\[?static\]?|\[?noise\]?)[ .,-]*$','',text).strip()

            # V087 stabilization: quarantine obvious Whisper loops and very weak/no-speech
            # decodes. Keep the raw text visible for review, but do not let it become
            # canonical incident evidence unless an operator corrects it.
            reject_reason = ''
            if text:
                words = re.findall(r"[a-z0-9']+", text.lower())
                # Detect a phrase/sentence that is repeated three or more times. Scanner
                # dispatch can legitimately repeat once, so the threshold is intentionally 3.
                for n in range(min(14, max(4, len(words)//3)), 3, -1):
                    if len(words) < n * 3:
                        continue
                    chunks = [tuple(words[i:i+n]) for i in range(0, len(words)-n+1)]
                    counts = {}
                    for c in chunks: counts[c] = counts.get(c, 0) + 1
                    if counts and max(counts.values()) >= 3:
                        reject_reason = 'Rejected probable Whisper repetition loop.'
                        break
                avg_logprob = payload.get('avg_logprob') if backend == 'faster_whisper' else None
                no_speech_prob = payload.get('no_speech_prob') if backend == 'faster_whisper' else None
                if not reject_reason and no_speech_prob is not None and float(no_speech_prob) >= 0.80:
                    reject_reason = f'Rejected probable no-speech hallucination (no_speech={float(no_speech_prob):.2f}).'
                if not reject_reason and avg_logprob is not None and float(avg_logprob) <= -1.50:
                    reject_reason = f'Rejected very low-confidence transcription (avg_logprob={float(avg_logprob):.2f}).'

            tx_status = 'HALLUCINATION_REJECTED' if reject_reason else ('TRANSCRIBED' if text else 'NO_SPEECH')
            con=db()
            try:
                con.execute("UPDATE transmissions SET transcript=?, transcript_status=?, transcript_error=?, transcribed_at=?, transcript_backend=? WHERE id=?", (text,tx_status,reject_reason,time.time(),backend_label,row['id'])); con.commit()
            finally: con.close()
            if text and not reject_reason: self.apply_duplicate_detection(row['id'])
            self.last_success=time.time(); self.count_session+=1; self.last_error=''
        except subprocess.TimeoutExpired:
            msg='Transcription timed out.'; self._set_status(row['id'],'RETRY',msg); self.last_error=msg
        except Exception as exc:
            msg=str(exc); self._set_status(row['id'],'RETRY',msg); self.last_error=msg
        finally:
            self.current_id=None

    @staticmethod
    def normalize_transcript(text: str) -> tuple[str, set[str]]:
        t = (text or '').lower()
        t = re.sub(r'[^a-z0-9\s]', ' ', t)
        t = re.sub(r'\b(uh|um|the|a|an|and|to|for|of)\b', ' ', t)
        t = re.sub(r'\s+', ' ', t).strip()
        return t, set(t.split())

    def apply_duplicate_detection(self, txid: int):
        cfg = load_config().get('duplicate_detection', {})
        if not cfg.get('enabled', True):
            return
        watcher_ids = {str(x) for x in cfg.get('watcher_ids', ['vox'])}
        window = float(cfg.get('window_seconds', 90))
        threshold = float(cfg.get('similarity_threshold', 0.78))
        min_words = int(cfg.get('min_words', 4))
        con = db()
        try:
            row = con.execute('SELECT * FROM transmissions WHERE id=?', (txid,)).fetchone()
            if not row or row['watcher_id'] not in watcher_ids or not (row['corrected_transcript'] or row['transcript']):
                return
            row_text = row['corrected_transcript'] or row['transcript']
            norm, words = self.normalize_transcript(row_text)
            if len(words) < min_words:
                return
            candidates = con.execute('''SELECT * FROM transmissions
                WHERE watcher_id=? AND id<>? AND (transcript IS NOT NULL OR corrected_transcript IS NOT NULL)
                  AND timestamp_epoch BETWEEN ? AND ?
                ORDER BY ABS(timestamp_epoch-?) ASC LIMIT 12''',
                (row['watcher_id'], txid, row['timestamp_epoch']-window, row['timestamp_epoch']+window, row['timestamp_epoch'])).fetchall()
            best = None
            best_score = 0.0
            for cand in candidates:
                cand_text = cand['corrected_transcript'] or cand['transcript']
                cnorm, cwords = self.normalize_transcript(cand_text)
                if len(cwords) < min_words:
                    continue
                seq = difflib.SequenceMatcher(None, norm, cnorm).ratio()
                union = words | cwords
                jac = (len(words & cwords) / len(union)) if union else 0.0
                score = max(seq, (seq * 0.65 + jac * 0.35))
                if score > best_score:
                    best_score = score
                    best = cand
            if best is None or best_score < threshold:
                return
            canonical_id = best['duplicate_of_id'] or best['id']
            group = best['duplicate_group'] or f'vox-repeat-{canonical_id}'
            # Prefer the longer transcript as canonical when the prior one was very short.
            best_text = best['corrected_transcript'] or best['transcript'] or ''
            if len(row_text) > len(best_text) * 1.35 and not best['duplicate_of_id']:
                canonical_id = row['id']
                group = f'vox-repeat-{canonical_id}'
                con.execute('UPDATE transmissions SET duplicate_group=?, duplicate_of_id=?, duplicate_similarity=? WHERE id=?',
                            (group, row['id'], best_score, best['id']))
                con.execute('UPDATE transmissions SET duplicate_group=?, duplicate_of_id=NULL, duplicate_similarity=NULL WHERE id=?',
                            (group, row['id']))
            else:
                con.execute('UPDATE transmissions SET duplicate_group=?, duplicate_of_id=?, duplicate_similarity=? WHERE id=?',
                            (group, canonical_id, best_score, row['id']))
                con.execute('UPDATE transmissions SET duplicate_group=COALESCE(duplicate_group,?) WHERE id=?', (group, canonical_id))
            con.commit()
        finally:
            con.close()

    def loop(self):
        # Old V066 records migrate to PENDING so archived test traffic can be transcribed too.
        con = db()
        try:
            con.execute("UPDATE transmissions SET transcript_status='PENDING' WHERE transcript_status IS NULL OR transcript_status='' OR transcript_status='MONITOR_ONLY'")
            con.commit()
        finally:
            con.close()
        while not STOP.wait(TRANSCRIBE_POLL_SECONDS):
            row = self._next_pending()
            if row:
                self.transcribe_one(row)

class DeepgramEngine:
    """Optional cloud comparison backend for the transcription bake-off."""
    def __init__(self):
        self.last_attempt = 0.0
        self.last_success = 0.0
        self.last_error = ''
        self.count_session = 0
        self.current_id = None

    def _api_key(self) -> str:
        env = (os.environ.get('DEEPGRAM_API_KEY') or '').strip()
        if env:
            return env
        try:
            if DEEPGRAM_KEY_PATH.exists():
                return DEEPGRAM_KEY_PATH.read_text(encoding='utf-8-sig').strip()
        except Exception:
            pass
        return ''

    def backend_status(self) -> dict:
        cfg = load_config().get('deepgram', {})
        key = self._api_key()
        return {
            'enabled': bool(cfg.get('enabled', True)),
            'ready': bool(cfg.get('enabled', True)) and bool(key),
            'configured': bool(key),
            'model': str(cfg.get('model') or 'nova-2-phonecall'),
            'setup_script': str(BASE_DIR / 'SETUP_DEEPGRAM.bat'),
            'key_path': str(DEEPGRAM_KEY_PATH),
        }

    def _next_pending(self):
        con = db()
        try:
            row = con.execute("""SELECT * FROM transmissions
                WHERE COALESCE(deepgram_status,'PENDING') IN ('PENDING','WAITING_SETUP','RETRY')
                ORDER BY timestamp_epoch ASC, id ASC LIMIT 1""").fetchone()
            return dict(row) if row else None
        finally:
            con.close()

    def _set_status(self, txid: int, status: str, error: str = ''):
        con = db()
        try:
            con.execute('UPDATE transmissions SET deepgram_status=?, deepgram_error=? WHERE id=?', (status, error, txid))
            con.commit()
        finally:
            con.close()

    def _audio_path(self, row: dict) -> Path | None:
        for candidate in (row.get('archive_path'), row.get('original_path')):
            if candidate:
                p = Path(candidate)
                if p.exists() and p.is_file():
                    return p
        return None

    def _keywords(self) -> list[str]:
        cfg = load_config().get('deepgram', {})
        out=[]; seen=set()
        for term in cfg.get('static_keywords', []) or []:
            t=str(term).strip()
            if len(t)>=3 and t.lower() not in seen:
                seen.add(t.lower()); out.append(t)
        try:
            con=db()
            rows=con.execute('SELECT phrase FROM vocabulary_phrases ORDER BY use_count DESC,last_seen DESC LIMIT 80').fetchall()
            con.close()
            for r in rows:
                for token in re.findall(r"[A-Za-z][A-Za-z'-]{2,}", r['phrase'] or ''):
                    k=token.lower()
                    if k not in seen:
                        seen.add(k); out.append(token)
                        if len(out)>=60: return out
        except Exception:
            pass
        return out[:60]

    def transcribe_one(self, row: dict):
        status=self.backend_status()
        if not status['enabled']:
            self._set_status(row['id'],'DISABLED'); return
        if not status['ready']:
            self._set_status(row['id'],'WAITING_SETUP','Run SETUP_DEEPGRAM.bat once and enter a Deepgram API key.')
            time.sleep(3); return
        audio=self._audio_path(row)
        if not audio:
            self._set_status(row['id'],'AUDIO_MISSING','Neither archived nor original audio file is available.'); return
        cfg=load_config().get('deepgram', {})
        params=[
            ('model', str(cfg.get('model') or 'nova-2-phonecall')),
            ('language', str(cfg.get('language') or 'en')),
            ('smart_format', 'true' if cfg.get('smart_format',True) else 'false'),
            ('punctuate', 'true' if cfg.get('punctuate',True) else 'false'),
            ('mip_opt_out', 'true' if cfg.get('mip_opt_out',True) else 'false'),
        ]
        boost=float(cfg.get('keyword_boost') or 2.0)
        for kw in self._keywords():
            params.append(('keywords', f'{kw}:{boost:g}'))
        url='https://api.deepgram.com/v1/listen?'+urllib.parse.urlencode(params, doseq=True)
        ctype='audio/mpeg' if audio.suffix.lower()=='.mp3' else ('audio/wav' if audio.suffix.lower()=='.wav' else (mimetypes.guess_type(str(audio))[0] or 'application/octet-stream'))
        req=urllib.request.Request(url, data=audio.read_bytes(), method='POST', headers={
            'Authorization':'Token '+self._api_key(),
            'Content-Type':ctype,
            'User-Agent':'NWALScanner-RadioIntelligence/0.70'
        })
        self.current_id=row['id']; self.last_attempt=time.time(); self._set_status(row['id'],'TRANSCRIBING')
        try:
            with urllib.request.urlopen(req, timeout=45) as resp:
                payload=json.loads(resp.read().decode('utf-8','replace'))
            alt=((((payload.get('results') or {}).get('channels') or [{}])[0].get('alternatives') or [{}])[0])
            text=re.sub(r'\s+',' ',str(alt.get('transcript') or '')).strip()
            conf=alt.get('confidence')
            try: conf=float(conf) if conf is not None else None
            except Exception: conf=None
            con=db()
            try:
                con.execute("""UPDATE transmissions SET deepgram_transcript=?, deepgram_status=?, deepgram_error='',
                    deepgram_transcribed_at=?, deepgram_model=?, deepgram_confidence=? WHERE id=?""",
                    (text,'TRANSCRIBED' if text else 'NO_SPEECH',time.time(),status['model'],conf,row['id']))
                con.commit()
            finally: con.close()
            self.last_success=time.time(); self.count_session+=1; self.last_error=''
        except urllib.error.HTTPError as exc:
            try: body=exc.read().decode('utf-8','replace')[-1200:]
            except Exception: body=''
            msg=f'Deepgram HTTP {exc.code}: {body or exc.reason}'
            state='WAITING_SETUP' if exc.code in (401,403) else 'RETRY'
            self._set_status(row['id'],state,msg); self.last_error=msg
            if state=='RETRY': time.sleep(2)
        except Exception as exc:
            msg=str(exc); self._set_status(row['id'],'RETRY',msg); self.last_error=msg; time.sleep(2)
        finally:
            self.current_id=None

    def loop(self):
        con=db()
        try:
            con.execute("UPDATE transmissions SET deepgram_status='PENDING' WHERE deepgram_status IS NULL OR deepgram_status='' OR deepgram_status='MONITOR_ONLY'")
            con.commit()
        finally: con.close()
        while not STOP.wait(DEEPGRAM_POLL_SECONDS):
            row=self._next_pending()
            if row: self.transcribe_one(row)


ENGINE = IngestEngine()
TRANSCRIBER = TranscriptionEngine()
DEEPGRAM = DeepgramEngine()


def audio_path_for_id(txid: int) -> Path | None:
    con = db()
    try:
        row = con.execute('SELECT archive_path, original_path FROM transmissions WHERE id=?', (int(txid),)).fetchone()
        if not row:
            return None
        for candidate in (row['archive_path'], row['original_path']):
            if candidate:
                p = Path(candidate)
                if p.exists() and p.is_file():
                    return p
        return None
    finally:
        con.close()


def _extract_vocabulary_phrases(raw: str, corrected: str) -> list[str]:
    raw_words = re.findall(r"[A-Za-z0-9'/-]+", raw or '')
    cor_words = re.findall(r"[A-Za-z0-9'/-]+", corrected or '')
    phrases = []
    sm = difflib.SequenceMatcher(None, [w.lower() for w in raw_words], [w.lower() for w in cor_words])
    for tag, i1, i2, j1, j2 in sm.get_opcodes():
        if tag == 'equal' or j1 == j2:
            continue
        a = max(0, j1 - 1); b = min(len(cor_words), j2 + 1)
        phrase = ' '.join(cor_words[a:b]).strip()
        if 2 <= len(phrase) <= 90:
            phrases.append(phrase)
    clean = re.sub(r'\s+', ' ', corrected or '').strip()
    if 2 <= len(clean) <= 140:
        phrases.append(clean)
    out=[]; seen=set()
    for ph in phrases:
        key=ph.lower()
        if key not in seen:
            seen.add(key); out.append(ph)
    return out[:8]


def save_correction(txid: int, corrected: str, who: str = 'operator') -> dict:
    corrected = re.sub(r'\s+', ' ', corrected or '').strip()
    if not corrected:
        raise ValueError('Corrected transcript cannot be blank.')
    if len(corrected) > 5000:
        raise ValueError('Corrected transcript is too long.')
    con = db()
    try:
        row = con.execute('SELECT id, transcript FROM transmissions WHERE id=?', (int(txid),)).fetchone()
        if not row:
            raise LookupError('Transmission not found.')
        raw = row['transcript'] or ''
        now = time.time()
        con.execute('UPDATE transmissions SET corrected_transcript=?, corrected_at=?, corrected_by=? WHERE id=?',
                    (corrected, now, who, int(txid)))
        con.execute('INSERT INTO transcript_corrections (transmission_id,raw_transcript,corrected_transcript,corrected_at,corrected_by) VALUES (?,?,?,?,?)',
                    (int(txid), raw, corrected, now, who))
        for phrase in _extract_vocabulary_phrases(raw, corrected):
            con.execute('''INSERT INTO vocabulary_phrases (phrase,first_seen,last_seen,use_count,source) VALUES (?,?,?,?,?)
                ON CONFLICT(phrase) DO UPDATE SET last_seen=excluded.last_seen, use_count=vocabulary_phrases.use_count+1''',
                (phrase, now, now, 1, 'operator_correction'))
        con.commit()
    finally:
        con.close()
    try:
        TRANSCRIBER.apply_duplicate_detection(int(txid))
    except Exception:
        pass
    return {'id': int(txid), 'corrected_transcript': corrected, 'corrected_at': now, 'corrected_by': who}


def correction_summary() -> dict:
    con = db()
    try:
        corrected = con.execute("SELECT COUNT(*) n FROM transmissions WHERE corrected_transcript IS NOT NULL AND TRIM(corrected_transcript)<>''").fetchone()['n']
        vocab = con.execute('SELECT COUNT(*) n FROM vocabulary_phrases').fetchone()['n']
        return {'corrected': corrected, 'vocabulary_phrases': vocab}
    finally:
        con.close()


def _match_pct(candidate: str, verified: str) -> float | None:
    if not candidate or not verified:
        return None
    def norm(x):
        return re.sub(r'\s+',' ',re.sub(r'[^a-z0-9\s]',' ',x.lower())).strip()
    a,b=norm(candidate),norm(verified)
    if not a or not b:
        return None
    return round(difflib.SequenceMatcher(None,a,b).ratio()*100,1)


def recent_transmissions(limit=100):
    con = db()
    try:
        rows = con.execute('SELECT * FROM transmissions ORDER BY timestamp_epoch DESC, id DESC LIMIT ?', (int(limit),)).fetchall()
        out=[]
        for r in rows:
            x=dict(r)
            if x.get('corrected_transcript'):
                x['whisper_match_pct']=_match_pct(x.get('transcript') or '',x['corrected_transcript'])
                x['deepgram_match_pct']=_match_pct(x.get('deepgram_transcript') or '',x['corrected_transcript'])
            else:
                x['whisper_match_pct']=None; x['deepgram_match_pct']=None
            out.append(x)
        return out
    finally:
        con.close()


_INCIDENT_STOPWORDS = {
    'the','a','an','and','or','to','of','for','on','in','at','with','from','be','is','are','was','were','this','that',
    'respond','response','dispatch','advised','advise','unit','units','engine','truck','rescue','ems','fire','police','county',
    'city','station','copy','received','traffic','radio','road','street','drive','highway','north','south','east','west'
}

_INCIDENT_PATTERNS = [
    ('Wreck with Injuries', [r'\bwreck\b.*\binjur', r'\baccident\b.*\binjur', r'\bvehicle\b.*\binjur', r'\bmva\b.*\binjur', r'\bmvc\b.*\binjur', r'\bmvc\b']),
    ('Motor Vehicle Accident', [r'\bwreck\b', r'\bmotor vehicle accident\b', r'\bvehicle accident\b', r'\bcrash\b']),
    ('Structure Fire', [r'\bstructure fire\b', r'\bhouse fire\b', r'\bresidential fire\b', r'\bcommercial fire\b', r'\bworking fire\b']),
    ('Vehicle Fire', [r'\bvehicle fire\b', r'\bcar fire\b', r'\btruck fire\b']),
    ('Brush / Woods Fire', [r'\bbrush fire\b', r'\bwoods fire\b', r'\bgrass fire\b']),
    ('Fire Alarm', [r'\bfire alarm\b', r'\balarm activation\b']),
    ('Medical Emergency', [r'\bmedical\b', r'\bpatient\b', r'\bambulance\b', r'\bems\b']),
    ('Traumatic Injury', [r'\btrauma\b', r'\btraumatic injury\b', r'\binjury\b']),
    ('Person Struck', [r'\bpedestrian\b.*\bstruck\b', r'\bperson struck\b', r'\bhit by (a )?(car|vehicle)\b']),
    ('Tree / Lines Down', [r'\btree down\b', r'\bpower lines? down\b', r'\blines? down\b']),
    ('Hazard / Gas Leak', [r'\bgas leak\b', r'\bcut gas line\b', r'\bhazmat\b']),
]

_ROUTINE_PATTERNS = [
    r'\b10[- ]?4\b', r'\bclear\b', r'\bavailable\b', r'\breturning to service\b', r'\bradio check\b', r'\btest page\b',
    r'\bdelayed due to train\b', r'\ben route\b$', r'\bon scene\b$', r'^copy\b', r'^received\b'
]

def _effective_text(row):
    return (row['corrected_transcript'] or row['transcript'] or '').strip()

def _clean_tokens(text):
    toks=re.findall(r"[a-z0-9']+", (text or '').lower())
    return {t for t in toks if len(t)>=3 and t not in _INCIDENT_STOPWORDS and not t.isdigit()}

def _classify_incident(text):
    t=(text or '').lower()
    for label, pats in _INCIDENT_PATTERNS:
        if any(re.search(p,t,re.I) for p in pats):
            return label
    return 'Radio Activity'

_INCIDENT_SPECIFICITY = {
    'Radio Activity': 0,
    'Medical Emergency': 10,
    'Traumatic Injury': 20,
    'Motor Vehicle Accident': 30,
    'Wreck with Injuries': 40,
    'Person Struck': 45,
    'Fire Alarm': 25,
    'Brush / Woods Fire': 30,
    'Vehicle Fire': 35,
    'Structure Fire': 40,
    'Tree / Lines Down': 35,
    'Hazard / Gas Leak': 35,
}

def _more_specific_incident(current, candidate):
    return candidate if _INCIDENT_SPECIFICITY.get(candidate, 1) > _INCIDENT_SPECIFICITY.get(current, 1) else current

def _is_routine(text):
    t=(text or '').strip().lower()
    if len(t.split()) < 4: return True
    return any(re.search(p,t,re.I) for p in _ROUTINE_PATTERNS)

def _extract_location(text):
    t=re.sub(r'\s+',' ',text or '').strip(' .')
    pats=[
        r'\b(?:at|near)\s+(.{4,80}?)(?=\s+(?:for|with|respond|time out|possible|and advise)\b|[.;]|$)',
        r'\b(?:intersection of|at the intersection of)\s+(.{4,80}?)(?=[.;]|$)',
        r'\b(?:on)\s+([A-Z0-9][^.;]{3,70}?(?:Road|Rd|Street|St|Drive|Dr|Boulevard|Blvd|Highway|Hwy|Lane|Ln|Avenue|Ave|Parkway|Pkwy))\b'
    ]
    for pat in pats:
        m=re.search(pat,t,re.I)
        if m:
            loc=m.group(1).strip(' ,.-')
            if 3 <= len(loc) <= 90:
                return loc
    return ''

def _group_match(group, item, cfg):
    dt=item['timestamp_epoch']-group['last_epoch']
    if dt < 0 or dt > float(cfg.get('max_gap_seconds',1200)): return 0
    shared=len(item['tokens'] & group['tokens'])
    same_type=item['incident_type']!='Radio Activity' and item['incident_type']==group['incident_type']
    same_source=item['source_name'] in group['sources']
    if shared >= 2: return 4 + min(shared,4)
    if same_type and shared >= 1 and dt <= float(cfg.get('same_type_window_seconds',900)): return 4
    if same_type and dt <= float(cfg.get('cross_source_fast_window_seconds',240)): return 3
    if same_source and shared >= 1 and dt <= 300: return 3
    return 0

def incident_candidates(limit=30):
    cfg=load_config().get('incident_grouping',{})
    if not cfg.get('enabled',True): return []
    cutoff=time.time()-float(cfg.get('lookback_hours',12))*3600
    con=db()
    try:
        rows=con.execute("""SELECT * FROM transmissions
            WHERE timestamp_epoch>=? AND duplicate_of_id IS NULL
              AND COALESCE(corrected_transcript, transcript, '')<>''
              AND (COALESCE(corrected_transcript,'')<>'' OR transcript_status='TRANSCRIBED')
            ORDER BY timestamp_epoch ASC, id ASC""",(cutoff,)).fetchall()
        dup_counts={r['duplicate_of_id']:r['n'] for r in con.execute("""SELECT duplicate_of_id,COUNT(*) n FROM transmissions
            WHERE duplicate_of_id IS NOT NULL AND timestamp_epoch>=? GROUP BY duplicate_of_id""",(cutoff,)).fetchall()}
    finally:
        con.close()
    items=[]
    for r in rows:
        text=_effective_text(r)
        if len(text.split()) < int(cfg.get('min_words',4)): continue
        items.append({'id':r['id'],'timestamp_epoch':r['timestamp_epoch'],'timestamp_text':r['timestamp_text'],
                      'source_name':r['source_name'],'text':text,'tokens':_clean_tokens(text),'incident_type':_classify_incident(text),
                      'location':_extract_location(text),'verified':bool(r['corrected_transcript']),'repeat_copies':int(dup_counts.get(r['id'],0))})
    groups=[]
    for item in items:
        if _is_routine(item['text']) and item['incident_type']=='Radio Activity': continue
        best=None; bestscore=0
        for g in reversed(groups[-25:]):
            score=_group_match(g,item,cfg)
            if score>bestscore: bestscore=score; best=g
        if best is None or bestscore<3:
            groups.append({'key':f"incident-{item['id']}",'first_epoch':item['timestamp_epoch'],'last_epoch':item['timestamp_epoch'],
                           'incident_type':item['incident_type'],'location':item['location'],'sources':{item['source_name']},
                           'tokens':set(item['tokens']),'members':[item]})
        else:
            best['members'].append(item); best['last_epoch']=item['timestamp_epoch']; best['sources'].add(item['source_name']); best['tokens'] |= item['tokens']
            # Always allow a corrected or later transmission to refine the candidate title.
            # Example: an initial generic "injury" can become "Wreck with Injuries" once a verified MVC dispatch is available.
            best['incident_type']=_more_specific_incident(best['incident_type'], item['incident_type'])
            if not best['location'] and item['location']: best['location']=item['location']
    out=[]
    for g in groups:
        if g['incident_type']=='Radio Activity' and len(g['members'])<2: continue
        repeats=sum(m['repeat_copies'] for m in g['members'])
        confidence=45
        if g['incident_type']!='Radio Activity': confidence+=20
        if len(g['members'])>=2: confidence+=12
        if len(g['sources'])>=2: confidence+=10
        if any(m['verified'] for m in g['members']): confidence+=8
        if g['location']: confidence+=5
        confidence=min(98,confidence)
        # Internal token sets are useful for matching, but Python sets are not JSON serializable.
        # Strip matcher-only fields before returning incident evidence to the dashboard/API.
        public_members=[]
        for m in g['members']:
            public_members.append({k:v for k,v in m.items() if k != 'tokens'})
        out.append({'id':g['key'],'title':g['incident_type'],'incident_type':g['incident_type'],'location':g['location'],
                    'first_time':datetime.fromtimestamp(g['first_epoch']).strftime('%Y-%m-%d %H:%M:%S'),
                    'last_time':datetime.fromtimestamp(g['last_epoch']).strftime('%Y-%m-%d %H:%M:%S'),
                    'first_epoch':g['first_epoch'],'last_epoch':g['last_epoch'],'confidence':confidence,
                    'sources':sorted(g['sources']),'transmission_count':len(g['members']),'repeater_copies':repeats,
                    'members':public_members})
    states=_handoff_state([x['id'] for x in out])
    for x in out:
        st=states.get(x['id']); x['sent_to_incident_command']=bool(st); x['sent_to_incident_command_at']=st.get('sent_at') if st else None
    out.sort(key=lambda x:x['last_epoch'], reverse=True)
    return out[:int(limit)]

def _incident_command_type(label: str) -> str:
    t=(label or '').lower()
    if 'wreck' in t or 'vehicle accident' in t or 'motor vehicle' in t or 'person struck' in t: return 'wreck'
    if 'structure fire' in t or 'vehicle fire' in t or 'brush' in t or 'woods fire' in t or 'fire alarm' in t: return 'fire'
    if 'medical' in t or 'traumatic' in t: return 'medical'
    if 'gas leak' in t or 'hazard' in t: return 'cutgas'
    if 'tree' in t: return 'tree_down'
    if 'lines down' in t: return 'power_lines'
    return 'other'


def _incident_command_city(sources: list[str]) -> str:
    src=set(sources or [])
    if 'Florence Fire' in src: return 'Florence'
    if 'Muscle Shoals Fire' in src: return 'Muscle Shoals'
    if 'Sheffield Fire' in src: return 'Sheffield'
    if 'Tuscumbia Fire' in src: return 'Tuscumbia'
    if 'Colbert Fire' in src: return 'Colbert County'
    if 'Lauderdale County VOX' in src: return 'Lauderdale County'
    return ''


def _incident_command_agencies(candidate: dict) -> list[str]:
    title=(candidate.get('incident_type') or '').lower(); src=' '.join(candidate.get('sources') or []).lower(); agencies=[]
    if 'fire' in title or 'fire' in src or 'gas leak' in title or 'hazard' in title: agencies.append('fire')
    if any(k in title for k in ('medical','injur','person struck','wreck')) or 'ems' in src: agencies.append('ems')
    if 'wreck' in title or 'person struck' in title: agencies.append('police')
    return list(dict.fromkeys(agencies))


def _handoff_state(candidate_ids: list[str]) -> dict:
    if not candidate_ids: return {}
    con=db()
    try:
        marks=','.join('?' for _ in candidate_ids)
        rows=con.execute(f'SELECT candidate_id,sent_at,incident_url FROM incident_handoffs WHERE candidate_id IN ({marks})',candidate_ids).fetchall()
        return {r['candidate_id']:{'sent_at':r['sent_at'],'incident_url':r['incident_url']} for r in rows}
    finally: con.close()


def _incident_command_location(candidate: dict) -> str:
    """Apply NWAL privacy/detail rules to the location sent to Incident Command.

    Medical and fire/hazard calls get road-level location only. Wrecks and
    tree/power-line calls may retain the exact address/intersection reported.
    """
    loc=re.sub(r'\s+',' ',candidate.get('location') or '').strip(' ,.-')
    if not loc: return ''
    title=(candidate.get('incident_type') or candidate.get('title') or '').lower()

    # These call types intentionally retain exact addresses/intersections.
    exact_ok=any(k in title for k in (
        'motor vehicle', 'wreck', 'person struck', 'tree / lines down',
        'tree down', 'lines down', 'power lines', 'powerline'
    ))
    if exact_ok: return loc

    # Medical/fire/hazard incidents are handed off at road level only.
    road_level=any(k in title for k in (
        'medical', 'traumatic injury', 'fire', 'gas leak', 'hazard'
    ))
    if not road_level: return loc

    # Prefer a county-road designation anywhere in the extracted location.
    m=re.search(r'\b(?:county\s+road|co(?:unty)?\.?\s*rd|cr)\s*[-#]?\s*(\d+[a-z]?)\b',loc,re.I)
    if m: return f'CR {m.group(1)}'

    # Remove a leading street number while preserving the road name.
    cleaned=re.sub(r'^\s*\d+[A-Za-z]?[-/]?\d*\s+', '', loc).strip(' ,.-')

    # If extraction included a landmark after the road, keep just the named road.
    road=r'Road|Rd|Street|St|Drive|Dr|Boulevard|Blvd|Highway|Hwy|Lane|Ln|Avenue|Ave|Parkway|Pkwy|Circle|Cir|Court|Ct|Trail|Trl|Way|Place|Pl'
    m=re.search(rf'^(.+?\b(?:{road})\b)',cleaned,re.I)
    if m: cleaned=m.group(1).strip(' ,.-')
    return cleaned or loc


def build_incident_handoff(candidate_id: str) -> dict:
    candidate=next((c for c in incident_candidates(100) if c.get('id')==candidate_id),None)
    if not candidate: raise LookupError('Incident candidate not found in the current lookback window.')
    # Handoff only pre-fills operator-facing incident fields. Status / Notes and Board Notes
    # intentionally remain blank for the operator to enter in Incident Command.
    payload={'ri_candidate_id':candidate_id,'type':_incident_command_type(candidate.get('incident_type') or candidate.get('title') or ''),'title':candidate.get('title') or candidate.get('incident_type') or 'Radio Incident','location':_incident_command_location(candidate),'city':_incident_command_city(candidate.get('sources') or []),'priority':'high' if int(candidate.get('confidence') or 0)>=90 else 'normal','agencies':_incident_command_agencies(candidate),'confidence':candidate.get('confidence',0),'first_time':candidate.get('first_time','')}
    from urllib.parse import urlencode
    query=urlencode({k:(json.dumps(v) if isinstance(v,list) else v) for k,v in payload.items() if v not in ('',None,[])})
    incident_url='/incident_command.html?ri_prefill=1&'+query; now=time.time()
    con=db()
    try:
        con.execute('INSERT INTO incident_handoffs(candidate_id,sent_at,incident_url,payload_json) VALUES(?,?,?,?) ON CONFLICT(candidate_id) DO UPDATE SET sent_at=excluded.sent_at, incident_url=excluded.incident_url, payload_json=excluded.payload_json',(candidate_id,now,incident_url,json.dumps(payload,ensure_ascii=False))); con.commit()
    finally: con.close()
    return {'candidate_id':candidate_id,'sent_at':now,'incident_url':incident_url,'payload':payload}


def source_summary():
    con = db()
    try:
        rows = con.execute('''SELECT source_name, COUNT(*) count, MAX(timestamp_epoch) last_epoch,
                              SUM(CASE WHEN transcript_status='TRANSCRIBED' THEN 1 ELSE 0 END) transcribed,
                              SUM(CASE WHEN deepgram_status='TRANSCRIBED' THEN 1 ELSE 0 END) deepgram_transcribed
                              FROM transmissions GROUP BY source_name ORDER BY count DESC''').fetchall()
        return [dict(r) for r in rows]
    finally:
        con.close()


def transcript_summary():
    con = db()
    try:
        rows = con.execute('SELECT COALESCE(transcript_status,\'PENDING\') status, COUNT(*) count FROM transmissions GROUP BY COALESCE(transcript_status,\'PENDING\')').fetchall()
        return {r['status']: r['count'] for r in rows}
    finally:
        con.close()


def watcher_status():
    cfg = load_config()
    out = []
    for w in cfg.get('watchers', []):
        p = expand_path(str(w.get('path') or ''))
        out.append({**w, 'resolved_path': str(p), 'exists': p.exists() and p.is_dir()})
    return out


HTML = r'''<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Radio Intelligence · Incident Intelligence Phase 1 · Resilient Dashboard</title><style>
:root{--bg:#06101a;--panel:#0d1b29;--line:#20384c;--text:#edf6ff;--muted:#8da7bb;--accent:#1da1f2;--good:#39d98a;--warn:#ffbd4a;--bad:#ff7272;--violet:#b69cff}
*{box-sizing:border-box}body{margin:0;background:linear-gradient(145deg,#050b12,#081521 55%,#04090e);color:var(--text);font-family:Inter,Segoe UI,Arial,sans-serif;min-height:100vh}.shell{max-width:1500px;margin:auto;padding:28px}.top{display:flex;justify-content:space-between;gap:18px;align-items:flex-start}.top h1{margin:0;font-size:32px}.top p{color:var(--muted);margin:6px 0}.badge{border:1px solid rgba(255,189,74,.4);background:rgba(255,189,74,.08);color:#ffd990;padding:9px 12px;border-radius:999px;font-weight:800}.section{margin-top:22px;background:rgba(13,27,41,.94);border:1px solid var(--line);border-radius:16px;padding:18px}.section h2{margin:0 0 14px}.engines,.watchers,.summary{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px}.card{background:#102438;border:1px solid var(--line);border-radius:12px;padding:14px}.muted{color:var(--muted);font-size:13px}.dot{display:inline-block;width:9px;height:9px;border-radius:50%;background:var(--warn);margin-right:7px}.dot.good{background:var(--good)}.dot.bad{background:var(--bad)}.setup{margin-top:12px;border:1px solid rgba(255,189,74,.35);background:rgba(255,189,74,.06);padding:12px 14px;border-radius:11px;color:#ffe1a2}.setup code{color:#fff}.transmissions{display:flex;flex-direction:column;gap:10px}.tx{border:1px solid var(--line);background:#091621;border-radius:12px;padding:13px 14px}.txhead{display:grid;grid-template-columns:170px 220px 1fr auto;gap:12px;align-items:center}.source{font-weight:800}.status{font-size:11px;font-weight:900;letter-spacing:.04em;color:#ffd990}.status.ok{color:#79e7ad}.status.wait{color:#ffd990}.status.err{color:#ff9292}.transcriptrow{display:flex;gap:10px;align-items:stretch;margin-top:10px}.transcript{flex:1;padding:11px 12px;border-left:3px solid #2a98d9;background:#0c1e2d;border-radius:7px;font-size:15px;line-height:1.45}.transcript.corrected{border-left-color:var(--good);background:#0d241f}.comparegrid{display:grid;grid-template-columns:1fr 1fr;gap:9px;margin-top:9px}.enginebox{background:#071724;border:1px solid #203e54;border-radius:8px;padding:9px 10px}.enginebox.dg{border-color:#53447a;background:#121427}.enginetitle{font-size:10px;font-weight:900;letter-spacing:.06em;color:#86cfff;margin-bottom:5px}.enginebox.dg .enginetitle{color:#c4b2ff}.confidence{font-size:11px;color:#b9c9d6;margin-left:7px}.match{font-size:11px;color:#79e7ad;margin-left:7px}.engineerr{font-size:11px;color:#ff9292;margin-top:5px}@media(max-width:900px){.comparegrid{grid-template-columns:1fr}}.label{display:inline-block;font-size:10px;font-weight:900;letter-spacing:.06em;color:#86cfff;margin-right:8px}.label.corrected{color:#79e7ad}.actions{display:flex;gap:8px;align-items:flex-start;flex-wrap:wrap}.btn{border:1px solid #31516a;background:#102a3f;color:#fff;padding:9px 12px;border-radius:8px;font-weight:800;cursor:pointer}.btn:hover{background:#17364f}.btn.playing{border-color:#ffbd4a;color:#ffd990}.btn.save{background:#145c43;border-color:#2b9c72}.btn.cancel{background:#202b35}.editor{margin-top:9px;display:none}.editor textarea{width:100%;min-height:82px;background:#07131e;color:#fff;border:1px solid #31516a;border-radius:8px;padding:10px;font:inherit;line-height:1.4}.editorbar{display:flex;gap:8px;margin-top:8px}.rawdetails{margin-top:8px}.rawdetails summary{color:var(--muted);cursor:pointer;font-size:12px}.rawdetails .raw{margin-top:7px;color:#bdd0df;font-size:13px;line-height:1.4;background:#07131e;border-radius:7px;padding:9px}.dup{display:inline-block;margin-left:8px;color:#d9ccff;border:1px solid rgba(182,156,255,.35);padding:2px 7px;border-radius:999px;font-size:10px;font-weight:900}.verified{display:inline-block;margin-left:8px;color:#79e7ad;border:1px solid rgba(57,217,138,.35);padding:2px 7px;border-radius:999px;font-size:10px;font-weight:900}.file{color:var(--muted);font-size:11px;word-break:break-all;margin-top:8px}.empty{color:var(--muted);padding:25px 0}.footer{margin-top:20px;color:var(--muted);font-size:12px}@media(max-width:900px){.engines,.watchers,.summary{grid-template-columns:1fr}.shell{padding:16px}.top{flex-direction:column}.txhead{grid-template-columns:1fr}.transcriptrow{flex-direction:column}}
.incidents{display:flex;flex-direction:column;gap:12px}.incident{background:#0a1926;border:1px solid #28506c;border-radius:13px;padding:14px}.incidenthead{display:grid;grid-template-columns:1fr auto;gap:12px;align-items:start}.incidenttitle{font-size:19px;font-weight:900}.incidentmeta{color:var(--muted);font-size:12px;margin-top:5px}.conf{display:inline-block;border:1px solid rgba(57,217,138,.35);color:#79e7ad;border-radius:999px;padding:4px 8px;font-size:11px;font-weight:900}.loc{margin-top:8px;color:#d8ecff;font-weight:700}.evidence{margin-top:10px;border-top:1px solid #1c374a;padding-top:9px}.evidence summary{cursor:pointer;color:#9fd8ff;font-weight:800}.evrow{display:grid;grid-template-columns:145px 170px 1fr auto;gap:8px;align-items:center;padding:8px 0;border-bottom:1px solid #142a39;font-size:13px}.evrow:last-child{border-bottom:0}.tinybtn{border:1px solid #31516a;background:#102a3f;color:#fff;padding:6px 9px;border-radius:7px;font-weight:800;cursor:pointer}.handoffbtn{border-color:#a86d14;background:#3a270b;color:#ffd990}.handoffbtn:hover{background:#53380d}.handoffbtn.sent{border-color:#2b9c72;background:#123b2e;color:#79e7ad}.tag{display:inline-block;margin-left:6px;color:#d9ccff;font-size:10px}@media(max-width:900px){.incidenthead,.evrow{grid-template-columns:1fr}}</style></head><body><div class="shell"><div class="top"><div><h1>Radio Intelligence</h1><p>Local audio ingest + scanner-tuned transcription + incident grouping + Incident Command handoff · Phase 7</p></div><div class="badge">MONITOR ONLY · NOTHING AUTO-POSTS</div></div>
<div class="section"><h2>Transcription Engines</h2><div class="engines"><div id="transcriber" class="card"></div></div><div id="setup" class="setup" style="display:none"></div></div>
<div class="section"><h2>Recording Sources</h2><div id="watchers" class="watchers"></div></div>
<div class="section"><h2>Source Activity</h2><div id="summary" class="summary"></div></div>
<div class="section"><h2>Incident Candidates</h2><div id="incidenterror" class="setup" style="display:none"></div><div id="incidents" class="incidents"></div><div id="noincidents" class="empty">No incident candidates in the current lookback window.</div></div>
<div class="section"><h2>Recent Transmissions</h2><div id="rows" class="transmissions"></div><div id="empty" class="empty">Waiting for new completed recordings…</div></div>
<div class="footer" id="footer"></div></div><script>
function e(s){return String(s??'').replace(/[&<>"']/g,m=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[m]))}
async function get(p){const r=await fetch(p,{cache:'no-store'});let j;try{j=await r.json()}catch(err){throw new Error(`${p}: invalid JSON response (${r.status})`)}if(!r.ok)throw new Error(j.error||`${p}: HTTP ${r.status}`);return j}
async function post(p,obj){const r=await fetch(p,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(obj)});const j=await r.json();if(!r.ok)throw new Error(j.error||'Request failed');return j}
function fmtBytes(n){if(n<1024)return n+' B';if(n<1048576)return (n/1024).toFixed(1)+' KB';return (n/1048576).toFixed(1)+' MB'}
function statusClass(s){if(s==='TRANSCRIBED'||s==='NO_SPEECH')return'ok';if(s==='WAITING_SETUP'||s==='PENDING'||s==='TRANSCRIBING')return'wait';return'err'}
let activeAudio=null,activeBtn=null;
function playTx(id,btn){if(activeAudio){activeAudio.pause();activeAudio=null;if(activeBtn){activeBtn.classList.remove('playing');activeBtn.textContent='▶ Play'}if(activeBtn===btn){activeBtn=null;return}}const a=new Audio('/api/audio/'+id);activeAudio=a;activeBtn=btn;btn.classList.add('playing');btn.textContent='■ Stop';a.onended=a.onerror=()=>{btn.classList.remove('playing');btn.textContent='▶ Play';activeAudio=null;activeBtn=null};a.play().catch(()=>{btn.classList.remove('playing');btn.textContent='▶ Play';activeAudio=null;activeBtn=null})}
const editingIds=new Set();
const incidentEditingIds=new Set();
function editTx(id){const ed=document.getElementById('ed-'+id);if(!ed)return;const opening=ed.style.display!=='block';if(opening){editingIds.add(id);ed.style.display='block';const ta=document.getElementById('ta-'+id);if(ta){requestAnimationFrame(()=>{ta.focus();ta.setSelectionRange(ta.value.length,ta.value.length)})}}else{editingIds.delete(id);ed.style.display='none'}}
async function saveTx(id){const ta=document.getElementById('ta-'+id);const b=document.getElementById('save-'+id);if(!ta||!b)return;b.disabled=true;b.textContent='Saving…';try{await post('/api/transmissions/'+id+'/correction',{corrected_transcript:ta.value});editingIds.delete(id);await refresh(true)}catch(err){alert(err.message)}finally{b.disabled=false;b.textContent='Save correction'}}
function editEvidenceTx(id,btn){const row=btn.closest('.evrow');if(!row)return;let ed=row.querySelector('.eveditor');if(!ed)return;const opening=ed.style.display!=='block';if(opening){incidentEditingIds.add(id);ed.style.display='block';const ta=ed.querySelector('textarea');if(ta){requestAnimationFrame(()=>{ta.focus();ta.setSelectionRange(ta.value.length,ta.value.length)})}}else{incidentEditingIds.delete(id);ed.style.display='none'}}
async function saveEvidenceTx(id,btn){const ed=btn.closest('.eveditor');if(!ed)return;const ta=ed.querySelector('textarea');if(!ta)return;btn.disabled=true;btn.textContent='Saving…';try{await post('/api/transmissions/'+id+'/correction',{corrected_transcript:ta.value});incidentEditingIds.delete(id);await refresh(true)}catch(err){alert(err.message)}finally{btn.disabled=false;btn.textContent='Save correction'}}
async function sendToIncidentCommand(id,btn){if(btn.dataset.sent==='1')return;const original=btn.textContent;btn.disabled=true;btn.textContent='Sending…';try{const r=await post('/api/incidents/'+encodeURIComponent(id)+'/handoff',{});btn.dataset.sent='1';btn.textContent='✓ Sent to Incident Command';btn.classList.add('sent');const rel=r.incident_url||'/incident_command.html?ri_prefill=1';const url=new URL(rel,window.location.href);url.port='9000';url.hostname=window.location.hostname;window.open(url.toString(),'_blank','noopener');await refresh(true)}catch(err){btn.disabled=false;btn.textContent=original;alert(err.message)}}
async function refresh(forceRows=false){
let s=null,t=null,ic=null;
let statusErr='',txErr='',incidentErr='';
try{s=await get('/api/status')}catch(err){statusErr=err.message||String(err)}
try{t=await get('/api/transmissions?limit=100')}catch(err){txErr=err.message||String(err)}
try{ic=await get('/api/incidents?limit=30')}catch(err){incidentErr=err.message||String(err)}

if(s){
 const tr=s.transcriber||{},dg=s.deepgram||{};
 transcriber.innerHTML=`<div><span class="dot ${tr.ready?'good':'bad'}"></span><b>${tr.ready?'GPU-Preferred Local Whisper Ready':'Local Transcription Setup Needed'}</b></div><div class="muted" style="margin-top:7px">Local/offline · ${e(tr.backend)} · primary ${e(tr.model)} / CUDA</div><div class="muted" style="margin-top:4px">Session: ${s.session_transcribed||0} ${s.current_transcription_id?'· Working on #'+s.current_transcription_id:''}</div>`;
 setup.style.display=tr.ready?'none':'block';setup.innerHTML=`Run <code>SETUP_RADIO_TRANSCRIPTION.bat</code> once if local Whisper is not installed.`;
 watchers.innerHTML=(s.watchers||[]).map(w=>`<div class="card"><div><span class="dot ${w.exists?'good':''}"></span><b>${e(w.name)}</b></div><div class="muted" style="margin-top:8px">${e(String(w.adapter||'').toUpperCase())}</div><div class="muted" style="margin-top:5px;word-break:break-all">${e(w.resolved_path)}</div><div style="margin-top:8px">${w.exists?'Ready':'Folder not found'}</div></div>`).join('');
 summary.innerHTML=(s.sources||[]).length?(s.sources||[]).map(x=>`<div class="card"><div class="source">${e(x.source_name)}</div><div style="font-size:26px;font-weight:800;margin-top:6px">${x.count}</div><div class="muted">indexed · Local transcripts ${x.transcribed||0}</div></div>`).join(''):'<div class="muted">No transmissions indexed yet.</div>';
 footer.textContent=`Updated ${s.timestamp} · Ingested ${s.session_ingested||0} · Whisper ${s.session_transcribed||0} · Corrections ${(s.corrections||{}).corrected||0} · Learned phrases ${(s.corrections||{}).vocabulary_phrases||0} · Archive ${s.archive_audio?'ON':'OFF'} · ${s.last_error?'Ingest: '+s.last_error:'No ingest errors'}${s.transcription_error?' · Whisper: '+s.transcription_error:''}${incidentErr?' · Incident API: '+incidentErr:''}`;
}else{
 transcriber.innerHTML=`<div><span class="dot bad"></span><b>Radio Intelligence status unavailable</b></div><div class="muted" style="margin-top:7px">${e(statusErr)}</div>`;
 watchers.innerHTML='<div class="muted">Status API unavailable. Recording-source monitoring may still be running server-side.</div>';
 summary.innerHTML='<div class="muted">Status API unavailable.</div>';
 footer.textContent='Dashboard status refresh error: '+statusErr;
}

const inc=document.getElementById('incidents'), noinc=document.getElementById('noincidents'), incerr=document.getElementById('incidenterror');
if(incidentErr){
 incerr.style.display='block';incerr.innerHTML=`<b>Incident grouping temporarily unavailable.</b><div class="muted" style="margin-top:5px">The rest of Radio Intelligence will continue updating. ${e(incidentErr)}</div>`;
 inc.innerHTML='';noinc.style.display='none';
}else if(ic){
 incerr.style.display='none';incerr.innerHTML='';
 // Preserve evidence-panel state across the dashboard's periodic refresh.
 // Incident IDs are anchored to the first canonical transmission, so they remain stable while a candidate grows.
 const openEvidenceIds=new Set(Array.from(inc.querySelectorAll('details.evidence[open]')).map(d=>d.dataset.incidentId).filter(Boolean));
 if(incidentEditingIds.size===0){inc.innerHTML=(ic.items||[]).map(g=>`<div class="incident" data-incident-id="${e(g.id)}"><div class="incidenthead"><div><div class="incidenttitle">${e(g.title)}</div><div class="incidentmeta">${e(g.first_time)} → ${e(g.last_time)} · ${g.transmission_count} canonical transmission${g.transmission_count===1?'':'s'}${g.repeater_copies?` · ${g.repeater_copies} repeater cop${g.repeater_copies===1?'y':'ies'} collapsed`:''} · ${e((g.sources||[]).join(' + '))}</div>${g.location?`<div class="loc">📍 ${e(g.location)}</div>`:''}</div><div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;justify-content:flex-end"><div class="conf">${g.confidence}% confidence</div><button class="tinybtn handoffbtn${g.sent_to_incident_command?' sent':''}" data-sent="${g.sent_to_incident_command?'1':'0'}" onclick="sendToIncidentCommand('${e(g.id)}',this)" ${g.sent_to_incident_command?'disabled':''}>${g.sent_to_incident_command?'✓ Sent to Incident Command':'Send to Incident Command →'}</button></div></div><details class="evidence" data-incident-id="${e(g.id)}"><summary>Review evidence (${(g.members||[]).length} transmission${(g.members||[]).length===1?'':'s'})</summary>${(g.members||[]).map(m=>`<div class="evrow"><div>${e(m.timestamp_text)}</div><div>${e(m.source_name)}</div><div>${e(m.text)}${m.verified?'<span class="tag">VERIFIED</span>':''}${m.repeat_copies?`<span class="tag">+${m.repeat_copies} repeater copies</span>`:''}<div class="eveditor" style="display:none;margin-top:8px"><textarea style="width:100%;min-height:72px">${e(m.text||'')}</textarea><div class="editorbar"><button class="tinybtn" onclick="saveEvidenceTx(${m.id},this)">Save correction</button><button class="tinybtn" onclick="editEvidenceTx(${m.id},this.closest('.evrow').querySelector('.evidence-correct'))">Cancel</button></div></div></div><div style="display:flex;gap:7px;justify-content:flex-end"><button class="tinybtn" onclick="playTx(${m.id},this)">▶ Play</button><button class="tinybtn evidence-correct" onclick="editEvidenceTx(${m.id},this)">✎ Correct</button></div></div>`).join('')}</details></div>`).join('');}
 inc.querySelectorAll('details.evidence').forEach(d=>{if(openEvidenceIds.has(d.dataset.incidentId))d.open=true;});
 noinc.style.display=(ic.items||[]).length?'none':'block';
}

if(t){
 const tr=(s&&s.transcriber)||{}, dg=(s&&s.deepgram)||{};
 if(forceRows||editingIds.size===0){rows.innerHTML=(t.items||[]).map(x=>{
  const verified=!!x.corrected_transcript;
  const seed=x.corrected_transcript||x.deepgram_transcript||x.transcript||'';
  const conf=x.deepgram_confidence==null?'':`<span class="confidence">confidence ${(x.deepgram_confidence*100).toFixed(1)}%</span>`;
  const wmatch=x.whisper_match_pct==null?'':`<span class="match">vs verified ${x.whisper_match_pct}%</span>`;
  const dmatch=x.deepgram_match_pct==null?'':`<span class="match">vs verified ${x.deepgram_match_pct}%</span>`;
  const whisperText=x.transcript||((x.transcript_status==='NO_SPEECH')?'[no speech]':'');
  const whisperBox=`<div class="enginebox"><div class="enginetitle">LOCAL FASTER-WHISPER · ${e(x.transcript_backend||tr.model||'')} ${wmatch}</div><div>${e(whisperText)}</div>${x.transcript_error?`<div class="engineerr">${e(x.transcript_error)}</div>`:''}</div>`;
  return `<div class="tx"><div class="txhead"><div>${e(x.timestamp_text)}</div><div class="source">${e(x.source_name)}</div><div class="muted">${e(x.adapter)} · ${fmtBytes(x.size_bytes)}</div><div><span class="status ${statusClass(x.transcript_status)}">LOCAL:${e(x.transcript_status||'PENDING')}</span>${verified?'<span class="verified">CORRECTED</span>':''}${x.duplicate_of_id?`<span class="dup">REPEATER COPY #${e(x.duplicate_of_id)}</span>`:''}</div></div>
  ${verified?`<div class="transcript corrected" style="margin-top:10px"><span class="label corrected">VERIFIED</span>${e(x.corrected_transcript)}</div>`:''}
  <div class="comparegrid">${whisperBox}</div>
  <div class="actions" style="margin-top:9px"><button class="btn" onclick="playTx(${x.id},this)">▶ Play</button><button class="btn" onclick="editTx(${x.id})">✎ Correct</button></div>
  <div class="editor" id="ed-${x.id}"><textarea id="ta-${x.id}">${e(seed)}</textarea><div class="editorbar"><button class="btn save" id="save-${x.id}" onclick="saveTx(${x.id})">Save correction</button><button class="btn cancel" onclick="editTx(${x.id})">Cancel</button></div></div>
  <div class="file">${e(x.filename)}</div></div>`}).join('');empty.style.display=(t.items||[]).length?'none':'block'}
}else{
 if(editingIds.size===0) rows.innerHTML=`<div class="setup"><b>Transmission list temporarily unavailable.</b><div class="muted" style="margin-top:5px">${e(txErr)}</div></div>`;
 empty.style.display='none';
}
}
refresh();setInterval(()=>refresh(false),2500)
</script></body></html>
'''

class Handler(BaseHTTPRequestHandler):
    def log_message(self, fmt, *args):
        return

    def json_out(self, code, obj):
        data = json.dumps(obj).encode('utf-8')
        self.send_response(code); self.send_header('Content-Type','application/json'); self.send_header('Cache-Control','no-store'); self.send_header('Content-Length',str(len(data))); self.end_headers(); self.wfile.write(data)

    def do_GET(self):
        u = urlparse(self.path)
        if u.path in ('/','/radio_intelligence.html'):
            data=HTML.encode('utf-8'); self.send_response(200); self.send_header('Content-Type','text/html; charset=utf-8'); self.send_header('Content-Length',str(len(data))); self.end_headers(); self.wfile.write(data); return
        if u.path == '/health':
            self.json_out(200, {'ok':True,'service':'radio-intelligence','phase':7.0,'monitor_only':True,'started':STARTED,'transcriber':TRANSCRIBER.backend_status(),'deepgram':DEEPGRAM.backend_status()}); return
        if u.path == '/api/status':
            cfg=load_config(); self.json_out(200, {'ok':True,'timestamp':datetime.now().strftime('%Y-%m-%d %H:%M:%S'),'tenant':TENANT_ID,'monitor_only':True,'watchers':watcher_status(),'sources':source_summary(),'transcript_counts':transcript_summary(),'session_ingested':ENGINE.count_session,'session_transcribed':TRANSCRIBER.count_session,'current_transcription_id':TRANSCRIBER.current_id,'session_deepgram':DEEPGRAM.count_session,'current_deepgram_id':DEEPGRAM.current_id,'deepgram_error':DEEPGRAM.last_error,'deepgram':DEEPGRAM.backend_status(),'last_scan':ENGINE.last_scan,'last_ingest':ENGINE.last_ingest,'last_error':ENGINE.last_error,'transcription_error':TRANSCRIBER.last_error,'transcriber':TRANSCRIBER.backend_status(),'archive_audio':cfg.get('archive_audio',True),'data_root':str(DATA_ROOT),'corrections':correction_summary()}); return
        if u.path == '/api/incidents':
            try:
                from urllib.parse import parse_qs
                try:
                    limit=max(1,min(100,int(parse_qs(u.query).get('limit',['30'])[0])))
                except Exception:
                    limit=30
                items=incident_candidates(limit)
                self.json_out(200, {'ok':True,'items':items,'count':len(items)})
            except Exception as exc:
                self.json_out(500, {'ok':False,'items':[],'error':'Incident grouping error: '+str(exc)})
            return
        if u.path == '/api/transmissions':
            try:
                from urllib.parse import parse_qs
                limit=max(1,min(500,int(parse_qs(u.query).get('limit',['100'])[0])))
            except Exception: limit=100
            self.json_out(200, {'items':recent_transmissions(limit)}); return
        m = re.fullmatch(r'/api/audio/(\d+)', u.path)
        if m:
            p = audio_path_for_id(int(m.group(1)))
            if not p:
                self.json_out(404, {'error':'audio not found'}); return
            size = p.stat().st_size
            ctype = 'audio/mpeg' if p.suffix.lower()=='.mp3' else 'audio/wav'
            rng = self.headers.get('Range','')
            start=0; end=size-1; code=200
            if rng.startswith('bytes='):
                try:
                    spec=rng[6:].split(',',1)[0]; a,b=spec.split('-',1)
                    if a: start=max(0,int(a))
                    if b: end=min(size-1,int(b))
                    if not a and b:
                        n=int(b); start=max(0,size-n); end=size-1
                    if start>end or start>=size: raise ValueError()
                    code=206
                except Exception:
                    self.send_response(416); self.send_header('Content-Range',f'bytes */{size}'); self.end_headers(); return
            length=end-start+1
            self.send_response(code); self.send_header('Content-Type',ctype); self.send_header('Accept-Ranges','bytes'); self.send_header('Content-Length',str(length))
            if code==206: self.send_header('Content-Range',f'bytes {start}-{end}/{size}')
            self.send_header('Cache-Control','no-store'); self.end_headers()
            with p.open('rb') as f:
                f.seek(start); remaining=length
                while remaining>0:
                    chunk=f.read(min(65536,remaining))
                    if not chunk: break
                    self.wfile.write(chunk); remaining-=len(chunk)
            return
        if u.path == '/api/config':
            self.json_out(200, load_config()); return
        self.json_out(404, {'error':'not found'})

    def do_POST(self):
        u=urlparse(self.path)
        correction=re.fullmatch(r'/api/transmissions/(\d+)/correction', u.path)
        handoff=re.fullmatch(r'/api/incidents/(incident-\d+)/handoff', u.path)
        if not correction and not handoff:
            self.json_out(404, {'error':'not found'}); return
        try:
            n=int(self.headers.get('Content-Length') or '0')
            if n<0 or n>20000: raise ValueError('Invalid request size.')
            payload=json.loads(self.rfile.read(n).decode('utf-8')) if n else {}
            if correction:
                result=save_correction(int(correction.group(1)), str(payload.get('corrected_transcript') or ''))
                self.json_out(200, {'ok':True, **result}); return
            result=build_incident_handoff(handoff.group(1)); self.json_out(200, {'ok':True, **result})
        except LookupError as exc: self.json_out(404, {'error':str(exc)})
        except (ValueError, json.JSONDecodeError) as exc: self.json_out(400, {'error':str(exc)})
        except Exception as exc: self.json_out(500, {'error':str(exc)})


def main():
    db().close()
    threading.Thread(target=ENGINE.loop, daemon=True, name='radio-ingest').start()
    threading.Thread(target=TRANSCRIBER.loop, daemon=True, name='radio-transcribe').start()
    threading.Thread(target=DEEPGRAM.loop, daemon=True, name='radio-deepgram').start()
    srv = ThreadingHTTPServer(('0.0.0.0', PORT), Handler)
    print(f'Radio Intelligence Phase 7 listening on http://0.0.0.0:{PORT}')
    print(f'Config: {CONFIG_PATH}')
    print(f'Data: {DATA_ROOT}')
    print(f'Whisper ready: {TRANSCRIBER.backend_status()["ready"]}')
    print(f'Deepgram ready: {DEEPGRAM.backend_status()["ready"]}')
    try:
        srv.serve_forever()
    except KeyboardInterrupt:
        pass
    finally:
        STOP.set(); srv.server_close()

if __name__ == '__main__':
    main()