"""Hermes Web UI -- Self-update checker. Checks if the webui and hermes-agent git repos are behind their latestrelease tags. Results are cached server-side (30-min TTL) so git fetch runsat most twice per hour regardless of client count. Skips repos that are not git checkouts (e.g. Docker baked images where.git does not exist)."""import hashlibimport jsonimport loggingimport osimport reimport shutilimport subprocessimport sysimport threadingimport timeimport urllib.errorimport urllib.requestfrom collections import OrderedDictfrom pathlib import Pathfrom urllib.parse import urlparse from api.gateway_restart import restart_active_profile_gatewayfrom api.config import REPO_ROOT, STREAMS, STREAMS_LOCK logger = logging.getLogger(__name__) # Lazy -- may be None if agent not foundtry:    from api.config import _AGENT_DIRexcept ImportError:    _AGENT_DIR = None _update_cache = {'webui': None, 'agent': None, 'checked_at': 0, 'include_agent': True, 'channel': 'stable'}_SUMMARY_CACHE_MAX = 16_summary_cache: OrderedDict = OrderedDict()_cache_lock = threading.Lock()_check_in_progress = False_apply_lock = threading.Lock()   # prevents concurrent stash/pull/pop on same repoCACHE_TTL = 1800  # 30 minutes_GIT_DIAGNOSTIC_MAX_CHARS = 300_CREDENTIAL_IN_URL_RE = re.compile(r"([a-zA-Z][a-zA-Z0-9+.-]*://)([^/@\s'\"]+)@")_GITHUB_TOKEN_RE = re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b")_QUERY_SECRET_RE = re.compile(r"([?&](?:access_token|oauth_token|private_token|client_secret|app_secret|api[_-]?key|token|password|secret|auth|key)=)[^&\s'\"]+", re.IGNORECASE)_FETCH_NETWORK_FAILURE_SIGNATURES = (    'could not resolve host',    'failed to connect',    'network is unreachable',    'no route to host',    'connection timed out',    'timed out after',    'connection reset by peer',    'remote end hung up unexpectedly',    'tls connection was non-properly terminated',    'ssl certificate problem',)_RELEASE_TAG_RE = re.compile(r'^v[0-9][0-9A-Za-z.+-]*$')# Phrases git emits when its own short-lived index/refs lock files block a# subsequent operation. Tuned to match only the true "lock file already exists"# semantics that warrant a lock-conflict response -- v2 deliberately drops the# broad "lock file" substring from the prior version to avoid false positives# on unrelated errors like "lock file lost during ref transaction".# Matched case-insensitively in _is_git_lock_error()._GIT_LOCK_SIGNATURES = (    "index.lock': file exists",    ".lock': file exists",    'another git process seems to be running',    'unable to create .git/index.lock',)# Lock files we previously enumerated for auto-removal in v2. v2.2 no longer# removes anything on the server, so the enumerable list is no longer needed;# ``_inventory_locks`` reports whatever ``.git/**/*.lock`` files currently exist# via plain ``rglob``.   def _sanitize_git_diagnostic(output: str, *, limit: int = _GIT_DIAGNOSTIC_MAX_CHARS) -> str:    """Return a user-facing git diagnostic with credentials removed.     Git can echo remote URLs in failure output.  Keep the actionable error text,    but strip URL userinfo, common GitHub token shapes, and secret-looking query    parameter values before any message reaches the update-check API/UI.    """    if not output:        return ""    sanitized = _CREDENTIAL_IN_URL_RE.sub(r"\1<redacted>@", str(output))    sanitized = _GITHUB_TOKEN_RE.sub("<redacted>", sanitized)    sanitized = _QUERY_SECRET_RE.sub(r"\1<redacted>", sanitized)    sanitized = sanitized.strip()    if len(sanitized) > limit:        sanitized = sanitized[:limit].rstrip() + "…"    return sanitized  def _apply_fetch_failure_message(fetch_out: str, network_message: str) -> str:    """Return the apply-path fetch failure message for the given stderr."""    detail = _sanitize_git_diagnostic(fetch_out)    if not detail:        return network_message    detail_lower = detail.lower()    if any(signature in detail_lower for signature in _FETCH_NETWORK_FAILURE_SIGNATURES):        return network_message    return f'fetch failed: {detail}'  def _restart_blocker_snapshot() -> dict:    """Return active chat work that should block a self-restart."""    with STREAMS_LOCK:        stream_ids = [str(k) for k in STREAMS.keys()]    run_ids: list[str] = []    try:        from api import config as _config        active_runs = getattr(_config, 'ACTIVE_RUNS', {})        active_runs_lock = getattr(_config, 'ACTIVE_RUNS_LOCK', None)        if active_runs_lock is not None:            with active_runs_lock:                run_ids = [str(k) for k in active_runs.keys()]        else:            run_ids = [str(k) for k in active_runs.keys()]    except Exception:        run_ids = []    return {        'active_streams': len(stream_ids),        'active_runs': len(run_ids),        'blocking_stream_ids': stream_ids[:10],        'blocking_run_ids': run_ids[:10],        'restart_blocked': bool(stream_ids or run_ids),    }  def _active_stream_count() -> int:    """Return the current in-memory chat stream count.     Kept for compatibility with older tests/helpers; restart safety should use    ``_restart_blocker_snapshot()`` so detached worker runs also block updates.    """    return int(_restart_blocker_snapshot().get('active_streams') or 0)  def _restart_blocked_response(target: str, blocker_snapshot: dict | int) -> dict:    if isinstance(blocker_snapshot, int):        blocker_snapshot = {            'active_streams': blocker_snapshot,            'active_runs': 0,            'blocking_stream_ids': [],            'blocking_run_ids': [],            'restart_blocked': bool(blocker_snapshot),        }    active_streams = int(blocker_snapshot.get('active_streams') or 0)    active_runs = int(blocker_snapshot.get('active_runs') or 0)    parts = []    if active_streams:        parts.append(f"{active_streams} active chat stream{'s' if active_streams != 1 else ''}")    if active_runs:        parts.append(f"{active_runs} active agent run{'s' if active_runs != 1 else ''}")    detail = ' and '.join(parts) or 'active chat work'    return {        'ok': False,        'message': (            f'Cannot update {target} while {detail} is running. '            'Wait for the response to finish, then retry the update.'        ),        'target': target,        'restart_blocked': True,        'active_streams': active_streams,        'active_runs': active_runs,        'blocking_stream_ids': blocker_snapshot.get('blocking_stream_ids') or [],        'blocking_run_ids': blocker_snapshot.get('blocking_run_ids') or [],    }  def _wait_until_restart_safe(poll_seconds: float = 2.0, max_wait_seconds: float = 300.0) -> dict:    """Wait for active work to finish before self-reexec.     Bounded by ``max_wait_seconds`` so a long-running (or stuck/orphaned) agent    run can't soft-jam the self-update indefinitely. If the deadline is reached    while work is still in flight, the snapshot is returned with    ``wait_timed_out=True`` so the caller can proceed with the re-exec anyway    (preserving the pre-#3105 "execv preempts in-flight work" fallback) rather    than holding ``_apply_lock`` for the run's full lifetime.    """    snapshot = _restart_blocker_snapshot()    deadline = time.monotonic() + max(0.0, max_wait_seconds)    while snapshot.get('restart_blocked'):        if time.monotonic() >= deadline:            logger.warning(                "restart-safety wait exceeded %.0fs with work still in flight (%s); "                "proceeding with re-exec anyway",                max_wait_seconds, snapshot,            )            snapshot = dict(snapshot)            snapshot['wait_timed_out'] = True            return snapshot        time.sleep(max(0.1, poll_seconds))        snapshot = _restart_blocker_snapshot()    return snapshot  def _run_git(args, cwd, timeout=10):    """Run a git command and return (useful output, ok).     On failure, returns stderr (or stdout as fallback) so callers can    surface actionable git error messages instead of empty strings.    """    git_executable = _resolve_git_executable()    if not git_executable:        return 'git executable not found', False    try:        r = subprocess.run(            [git_executable] + args, cwd=str(cwd), capture_output=True,            text=True, timeout=timeout,            encoding='utf-8', errors='replace',        )        # On non-UTF-8 locales (e.g. Chinese Windows GBK), a binary git        # output that fails to decode used to leave r.stdout = None and crash        # the whole import with AttributeError. Guard against None defensively.        stdout = (r.stdout or '').strip()        stderr = (r.stderr or '').strip()        if r.returncode == 0:            return stdout, True        return stderr or stdout or f"git exited with status {r.returncode}", False    except subprocess.TimeoutExpired as exc:        detail = (getattr(exc, 'stderr', None) or getattr(exc, 'stdout', None) or '').strip()        return detail or f"git {' '.join(args)} timed out after {timeout}s", False    except FileNotFoundError:        return 'git executable not found', False    except OSError as exc:        return f'git failed to start: {exc}', False  def _is_git_lock_error(output: str) -> bool:    if not output:        return False    lower_out = output.lower()    return any(sig in lower_out for sig in _GIT_LOCK_SIGNATURES)  def _inventory_locks(path: Path) -> dict:    """Return a snapshot of lock files currently present under ``path/.git``.     v2.2: replaced v2's `_is_lock_held` + `_try_remove_lock` machinery with    pure inventory. Round-2 cert (gate-fail) proved that `fcntl.flock`    cannot detect a live git lock, because git uses `O_CREAT|O_EXCL` and    `rename(2)`, NOT advisory locking. Any auto-delete path can therefore    race against a running `git add` and corrupt the index. v2.2 stops    deleting locks from the server entirely: the only thing that removes    a lock is the user, on the host, via the manual command surfaced in    the response. Once the lock is gone, the user re-clicks Update Now    and the normal non-destructive apply path runs.    """    git_dir = path / '.git'    out = {        'well_known_lock_present': False,  # ``.git/index.lock`` exists?        'well_known_lock_path': None,      # absolute path of ``.git/index.lock``        'other_locks': [],                  # any other lock files, by relative path    }    if not git_dir.exists():        return out    well_known = git_dir / 'index.lock'    try:        out['well_known_lock_present'] = well_known.exists()    except OSError:        # Permission problem reading the directory -- treat conservatively.        out['well_known_lock_present'] = True    out['well_known_lock_path'] = str(well_known)     # Enumerate every other lock file under .git/ for diagnostic reporting.    # We never touch them; this is purely an inventory.    try:        for entry in sorted(git_dir.rglob('*.lock')):            try:                rel = entry.relative_to(git_dir).as_posix()            except ValueError:                continue            if rel == 'index.lock':                continue            out['other_locks'].append(rel)    except OSError:        # rglob can fail on unreadable subtrees; skip quietly.        pass    return out  def apply_clear_lock(target: str) -> dict:    """Manual-instruction lock recovery for ``target``.     v2.2: NEVER removes a lock file. Strategy:       - If ``.git/index.lock`` is absent: re-run the normal non-destructive        apply path so the user lands on the latest version without ever        touching destructive git operations.      - If ``.git/index.lock`` is present: do NOT touch it -- the server        has no reliable proof that no live git process is still using        it (round-2 cert showed `fcntl.flock` does not detect git's        actual ``O_CREAT|O_EXCL`` locking). Return a response with the        exact manual command the operator can run, plus the inventory of        any other lock files so they can investigate. The frontend then        surfaces a copyable ``rm`` line and a "I've removed the lock --        try update again" button that re-invokes this endpoint, which        (now that the lock is gone) will take the success branch and        re-run the normal apply.    """    blocker_snapshot = _restart_blocker_snapshot()    if blocker_snapshot.get('restart_blocked'):        return _restart_blocked_response(target, blocker_snapshot)     if not _apply_lock.acquire(blocking=False):        return {'ok': False, 'message': 'Update already in progress'}     try:        if target == 'webui':            path = REPO_ROOT        elif target == 'agent':            path = _AGENT_DIR        else:            return {'ok': False, 'message': f'Unknown target: {target}'}         if path is None or not (path / '.git').exists():            return {'ok': False, 'message': 'Not a git repository'}         inv = _inventory_locks(path)        manual_command = f"rm -f {inv['well_known_lock_path']}"         if not inv['well_known_lock_present']:            # Lock is gone. Run the normal non-destructive update flow and            # annotate the response with what we found for the user's            # records. Pass the configured channel through — otherwise an            # experimental-channel WebUI lock-recovery retry silently falls back            # to stable (Codex gate: _apply_update_inner defaults to stable).            with _cache_lock:                _update_cache['checked_at'] = 0            retry_result = _apply_update_inner(target, _read_update_channel())            retry_result = dict(retry_result)            retry_result['lock_recovery'] = {                'action': 'no-lock-found',                'manual_command': manual_command,                'other_locks': inv['other_locks'],            }            return retry_result         # Lock is present. The server cannot prove it's safe to delete;        # the only safe path is to ask the operator.        message = (            'A git lock file (.git/index.lock) is present. The server does '            'not delete locks automatically -- git uses O_CREAT|O_EXCL '            'locking, which cannot be detected with advisory probes. To '            'recover: confirm no other git process is running against '            f'this checkout, then run: {manual_command}  '            'Click "Retry update" once you have removed it.'        )        return {            'ok': False,            'message': message,            'lock_held': True,            'target': target,            'manual_command': manual_command,            'well_known_lock_path': inv['well_known_lock_path'],            'other_locks': inv['other_locks'],        }    finally:        _apply_lock.release()  def _windows_git_from_registry():    """Best-effort resolve git.exe from the Git-for-Windows registry key.     Git for Windows records its install root at    ``HKLM\\SOFTWARE\\GitForWindows\\InstallPath`` (and the WOW6432Node mirror    for a 32-bit install on 64-bit Windows). ``git.exe`` lives under    ``<InstallPath>\\cmd\\git.exe``. This is the reliable way to find git when    it is installed but NOT on the launching process's PATH — e.g. the WebUI    server started from a venv python whose environment does not inherit the    interactive shell PATH, which otherwise degrades WEBUI_VERSION to    ``'unknown'`` and freezes the ``?v=`` static-asset cache-busting stamp.    """    try:        import winreg    except ImportError:        return None    for hive, flag in (        (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_64KEY),        (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_32KEY),        (winreg.HKEY_CURRENT_USER, 0),    ):        try:            with winreg.OpenKey(                hive, r'SOFTWARE\GitForWindows', 0,                winreg.KEY_READ | flag,            ) as key:                install_path, _ = winreg.QueryValueEx(key, 'InstallPath')        except OSError:            continue        if not install_path:            continue        candidate = os.path.join(install_path, 'cmd', 'git.exe')        if os.path.exists(candidate):            return candidate    return None  def _resolve_git_executable():    git_executable = shutil.which('git')    if git_executable:        return git_executable    if sys.platform == 'darwin' and os.path.exists('/usr/bin/git'):        return '/usr/bin/git'    if sys.platform == 'win32':        from_registry = _windows_git_from_registry()        if from_registry:            return from_registry        for candidate in (            os.path.expandvars(r'%ProgramFiles%\Git\cmd\git.exe'),            os.path.expandvars(r'%ProgramFiles(x86)%\Git\cmd\git.exe'),            os.path.expandvars(r'%LocalAppData%\Programs\Git\cmd\git.exe'),        ):            if candidate and os.path.exists(candidate):                return candidate    return None  def _dirty_suffix(path: Path, timeout=1) -> str:    """Return a best-effort ``-dirty`` suffix without blocking version display."""    out, ok = _run_git(['diff-index', '--quiet', 'HEAD', '--'], path, timeout=timeout)    if ok:        return ""    # diff-index --quiet exits 1 with no stdout/stderr to *signal* a dirty tree    # (not an error). _run_git() substitutes a synthetic "git exited with    # status N" diagnostic when both streams are empty, which makes the naive    # `if not out` guard always false on dirty trees — silently dropping the    # suffix and defeating dev-build cache busting (static/foo.js?v=… stays    # identical to the last-committed version). Treat the synthetic shape as    # the dirty signal; real errors (timeouts, missing git) carry a different    # diagnostic and correctly suppress the suffix.    if not out or out.startswith('git exited with status '):        diff, diff_ok = _run_git(['diff', '--binary', 'HEAD', '--'], path, timeout=timeout)        if diff_ok and diff:            digest = hashlib.sha1(diff.encode('utf-8', errors='replace')).hexdigest()[:8]            return f"-dirty-{digest}"        return "-dirty"    return ""  def _describe_git_version(path: Path, *, timeout=5, dirty_timeout=1) -> str | None:    """Return a fast git version string for a checkout, if available."""    out, ok = _run_git(['describe', '--tags', '--always'], path, timeout=timeout)    if not (ok and out):        return None    return out + _dirty_suffix(path, timeout=dirty_timeout)  def _detect_webui_version() -> str:    """Detect the running WebUI version from git or a baked-in fallback file.     Resolution order:      1. ``git describe --tags --always --dirty`` — works in any git checkout.         Returns the exact tag on tagged commits (e.g. ``v0.50.124``), a         post-tag descriptor between releases (e.g. ``v0.50.124-1-ge91325d``),         or a bare SHA when no tags exist (shallow clones, fresh forks).      2. ``api/_version.py`` — a fallback written by the Docker / CI release         workflow when ``.git`` is not present in the image.  Expected to define         ``__version__ = 'vX.Y.Z'``.      3. ``'unknown'`` — last resort; displayed as-is in the settings badge.    """    # Timeout capped at 3s: git describe on a healthy local repo is <50ms;    # a 10s stall on import (NFS-mounted .git, broken git binary) is unacceptable.    out = _describe_git_version(REPO_ROOT)    if out:        return out     # Docker / baked-image fallback: api/_version.py written by CI at build time.    # Parse with regex rather than exec() — the file holds exactly one assignment    # and regex is sufficient; exec() on a build artifact is an unnecessary surface.    version_file = REPO_ROOT / 'api' / '_version.py'    if version_file.exists():        try:            import re as _re            m = _re.search(                r"""__version__\s*=\s*['"]([^'"]+)['"]""",                version_file.read_text(encoding='utf-8'),            )            if m:                return m.group(1)        except Exception:            pass     return 'unknown'  def _read_agent_source_version(agent_dir: Path) -> str | None:    """Read Hermes Agent's package version from a copied source tree."""    init_file = agent_dir / 'hermes_cli' / '__init__.py'    try:        text = init_file.read_text(encoding='utf-8')    except (OSError, UnicodeDecodeError):        return None    m = re.search(r"""__version__\s*=\s*['"]([^'"]+)['"]""", text)

Lines 1–500 of 2342. Load more