from __future__ import annotations import argparseimport jsonimport osimport subprocessimport sysfrom pathlib import Path from . import benchmarks, config, sourcefrom .baseline import (    command_baseline_list,    command_baseline_record,    command_baseline_show,)from .common import (    AppNotDeployedError,    command_exists,    default_namespace,    deploy_app,    deployed_function,    print_json,)from .launch import command_build, command_rejudge, command_run, command_swe_atlasfrom .report import command_reportfrom .volume import (    command_builds,    command_fetch,    command_list,    command_logs,    command_runs,    command_status,    command_suite_fetch,    command_suite_logs,    command_suite_status,    ensure_volume_exists,)  def modal_secret_names() -> set[str]:    result = subprocess.run(        ["modal", "secret", "list", "--json"],        check=True,        capture_output=True,        text=True,    )    data = json.loads(result.stdout)    if not isinstance(data, list):        return set()    return {        item["name"] for item in data if isinstance(item, dict) and item.get("name")    }  def command_doctor(args: argparse.Namespace) -> int:    namespace = default_namespace(args)    print("zed-eval doctor")    print(f"  namespace:          {namespace}")    print(f"  app:                {args.app_name}")    print(f"  volume:             {args.volume}")    print(f"  api secret:         {args.api_secret}")    print(f"  modal token secret: {args.modal_token_secret}")    print(f"  repo root:          {source.repo_root()}")    print(f"  base sha:           {source.current_base_sha()}")    print(f"  default model:      {config.DEFAULT_MODEL}")    print("  default judges:     qna=deepseek-v4-pro, rf/tw=kimi-k2.7-code")     missing = []    for executable in ("git", "modal", "harbor"):        if command_exists(executable):            print(f"  {executable}:             found")        else:            print(f"  {executable}:             missing")            missing.append(executable)     if missing:        print("\nMissing required executables: " + ", ".join(missing), file=sys.stderr)        return 1     try:        secrets = modal_secret_names()    except (subprocess.CalledProcessError, json.JSONDecodeError) as error:        print(            f"  secrets:          could not list Modal secrets: {error}",            file=sys.stderr,        )    else:        missing_secrets = [            secret_name            for secret_name in (args.api_secret, args.modal_token_secret)            if secret_name not in secrets        ]        if missing_secrets:            print(                "\nMissing Modal secret(s): " + ", ".join(missing_secrets),                file=sys.stderr,            )            print(                "Use --api-secret/--modal-token-secret to point at existing secrets, "                "or create the missing ones with `modal secret create`.",                file=sys.stderr,            )            return 1        print("  secrets:          found")        print(            "  controller token: use a dedicated Modal service-user token for production"        )     if args.create_volume:        return ensure_volume_exists(args)    return 0  def command_deploy(args: argparse.Namespace) -> int:    deploy_app(args)    print(f"Deployed Modal app '{args.app_name}'.")    return 0  def command_cleanup(args: argparse.Namespace) -> int:    request: dict[str, object] = {"dry_run": bool(args.dry_run)}    if args.build_retention_days is not None:        request["build_retention_days"] = args.build_retention_days    result = deployed_function(args, "cleanup_artifacts").remote(request)    print_json(result)    return 0  def env_default(    variable_name: str, default: object, *, include_defaults: bool = True) -> object:    if not include_defaults:        return argparse.SUPPRESS    return os.environ.get(variable_name, default)  def add_common_options(    parser: argparse.ArgumentParser, *, include_defaults: bool = True) -> None:    default_volume = env_default(        "AGENT_EVALS_VOLUME",        config.DEFAULT_VOLUME_NAME,        include_defaults=include_defaults,    )    default_namespace = None if include_defaults else argparse.SUPPRESS    default_api_secret = env_default(        "AGENT_EVALS_LLM_PROVIDERS_SECRET",        config.DEFAULT_LLM_PROVIDERS_SECRET_NAME,        include_defaults=include_defaults,    )    default_modal_token_secret = env_default(        "AGENT_EVALS_MODAL_TOKEN_SECRET",        config.DEFAULT_MODAL_TOKEN_SECRET_NAME,        include_defaults=include_defaults,    )    default_app_name = env_default(        "AGENT_EVALS_APP_NAME",        config.DEFAULT_APP_NAME,        include_defaults=include_defaults,    )    infra = parser.add_argument_group(        "infra options", "Modal app/volume/namespace and secret names (rarely changed)"    )    infra.add_argument(        "--app-name",        default=default_app_name,        help=f"Deployed Modal app name (default: {config.DEFAULT_APP_NAME})",    )    infra.add_argument(        "--volume",        default=default_volume,        help=f"Modal volume for builds/runs (default: {config.DEFAULT_VOLUME_NAME})",    )    infra.add_argument(        "--namespace",        default=default_namespace,        help="Run namespace inside the shared volume (default: env/git user/local user)",    )    infra.add_argument(        "--api-secret",        default=default_api_secret,        help=(            "Modal secret with the LLM provider keys mounted into trial sandboxes "            f"(default: {config.DEFAULT_LLM_PROVIDERS_SECRET_NAME})"        ),    )    infra.add_argument(        "--modal-token-secret",        default=default_modal_token_secret,        help=(            "Modal secret mounted into the controller with MODAL_TOKEN_ID and MODAL_TOKEN_SECRET; "            f"use a dedicated service-user token for production (default: {config.DEFAULT_MODAL_TOKEN_SECRET_NAME})"        ),    )  def add_build_source_options(parser: argparse.ArgumentParser) -> None:    parser.add_argument(        "--from",        dest="from_source",        default=os.environ.get("AGENT_EVALS_FROM"),        help=(            "Unified build source: 'local' for current HEAD + tracked changes, "            "'main' for origin/main's tip, or a git ref/tag/SHA (e.g. v0.210.0) "            "for a clean build resolved canonically against the remote so "            "teammates share one build"        ),    )    parser.add_argument("--base-sha")    parser.add_argument("--patch-path")    parser.add_argument(        "--repo-url",        default=os.environ.get("AGENT_EVALS_REPO_URL", source.DEFAULT_REPO_URL),        help="Git remote the Modal builder fetches the base SHA from",    )    parser.add_argument(        "--allow-untracked",        action="store_true",        help="Proceed even when untracked files exist; they are not included in the build patch",    )    parser.add_argument(        "--require-clean",        action="store_true",        help="Fail if tracked changes are present",    )    parser.add_argument(        "--clean-source",        action="store_true",        help="Build exactly --base-sha/--zed-version with no local patch",    )    parser.add_argument(        "--zed-version",        help="Git ref/tag/SHA of Zed to build as a clean source snapshot",    )  def add_model_options(parser: argparse.ArgumentParser) -> None:    parser.add_argument(        "-m",        "--model",        default=os.environ.get("AGENT_EVALS_MODEL", config.DEFAULT_MODEL),        help="Base model as provider/model, sonnet-4.6, baseten:kimi-k2.7-code, or baseten:<model-id>",    )    parser.add_argument(        "--extra-api-secret",        action="append",        help="Additional Modal secret name to mount into the trial sandbox",    )    advanced = parser.add_argument_group(        "advanced model options",        "Lower-level model routing. For Baseten, prefer --model baseten:<model-id>.",    )    advanced.add_argument(        "--model-provider",        choices=("zed", "baseten"),        default="zed",        help="Use 'zed' for built-in provider/model ids, or 'baseten' for Baseten Model APIs",    )    advanced.add_argument(        "--baseten-model",        help="Baseten model id when --model-provider baseten, e.g. moonshotai/Kimi-K2.7-Code",    )    advanced.add_argument("--baseten-model-display-name")    advanced.add_argument(        "--baseten-api-url",        default=config.BASETEN_API_URL,        help=f"Baseten OpenAI-compatible API URL (default: {config.BASETEN_API_URL})",    )    advanced.add_argument(        "--baseten-model-max-tokens",        type=int,        default=config.BASETEN_DEFAULT_MAX_TOKENS,    )    advanced.add_argument(        "--baseten-model-max-output-tokens",        type=int,        default=config.BASETEN_DEFAULT_MAX_OUTPUT_TOKENS,    )    advanced.add_argument(        "--openai-compatible-provider-json",        help="JSON object merged into language_models.openai_compatible before eval-cli resolves --model",    )    advanced.add_argument(        "--anthropic-available-models-json",        help="JSON array merged into language_models.anthropic.available_models before eval-cli resolves --model",    )  def add_launch_options(parser: argparse.ArgumentParser) -> None:    add_model_options(parser)    parser.add_argument(        "-j",        "--judge",        default=os.environ.get("AGENT_EVALS_JUDGE", config.DEFAULT_JUDGE_PRESET),        choices=["auto", *sorted(config.JUDGES)],        help="Judge preset; auto uses qna=deepseek-v4-pro and rf/tw=kimi-k2.7-code",    )    parser.add_argument("--judge-model")    parser.add_argument(        "--build",        metavar="ID",        help="Reuse this build if it exists, otherwise create it with this id",    )    add_build_source_options(parser)    parser.add_argument("--build-wait-timeout-secs", type=int, default=7200)    parser.add_argument("--tasks", help="File containing one full task name per line")    parser.add_argument("--include-task-name", action="append")    parser.add_argument("-n", "--n-tasks", type=int, help="Forward harness --n-tasks")    parser.add_argument("--n-concurrent", type=int, default=config.DEFAULT_N_CONCURRENT)    parser.add_argument(        "--override-cpus", type=int, default=config.DEFAULT_OVERRIDE_CPUS    )    parser.add_argument(        "--override-memory-mb", type=int, default=config.DEFAULT_OVERRIDE_MEMORY_MB    )    parser.add_argument(        "--sandbox-timeout-secs", type=int, default=config.DEFAULT_SANDBOX_TIMEOUT_SECS    )    parser.add_argument(        "--sandbox-idle-timeout-secs",        type=int,        default=config.DEFAULT_SANDBOX_IDLE_TIMEOUT_SECS,    )    parser.add_argument(        "--eval-cli-timeout",        type=int,        help="Override the per-task agent timeout (defaults to the benchmark's)",    )    parser.add_argument("--run-id")    parser.add_argument(        "--dry-run",        action="store_true",        help="Print full manifests and harness commands without launching",    )    parser.add_argument(        "--plan",        action="store_true",        help="Print a concise launch plan without launching",    )    parser.add_argument(        "--verbose",        action="store_true",        help="With --plan, include full manifests and harness commands",    )    parser.add_argument(        "--extra-harbor-arg",        action="append",        help="Append one raw argument to the harness run command",    )    parser.add_argument(        "--swe-atlas-repo-url",        default=os.environ.get(            "AGENT_EVALS_SWE_ATLAS_REPO_URL", benchmarks.SWE_ATLAS_REPO_URL        ),        help="SWE-Atlas repo URL used for path-backed datasets",    )    parser.add_argument(        "--swe-atlas-repo-ref",        default=os.environ.get(            "AGENT_EVALS_SWE_ATLAS_REPO_REF", benchmarks.SWE_ATLAS_REPO_REF        ),        help="SWE-Atlas repo ref used for path-backed datasets",    )  def add_suite_options(parser: argparse.ArgumentParser) -> None:    add_launch_options(parser)    parser.add_argument(        "--parts",        help="Comma-separated SWE-Atlas parts to run: qna,rf,tw (or 'all')",    )    parser.add_argument("--experiment-prefix", default="swe-atlas")    parser.add_argument("--run-id-prefix")    parser.add_argument(        "--suite-id", help="Explicit suite id for grouping multi-part runs"    )    parser.add_argument("--interactive", action="store_true")    parser.add_argument(        "-y",        "--yes",        action="store_true",        help="Never prompt; require flags/defaults",    )  def add_run_lookup_options(    parser: argparse.ArgumentParser,    *,    require_experiment: bool = False,    run_id_optional: bool = False,) -> None:    if run_id_optional:        parser.add_argument(            "run_id",            nargs="?",            help="Run id (defaults to the most recent run launched from this machine)",        )    else:        parser.add_argument("run_id")    parser.add_argument(        "-e",        "--experiment-name",        required=require_experiment,        help="Benchmark storage name (auto-resolved from the local run index when omitted)",    )  def add_command_parser(    subparsers,    name: str,    *,    func: object | None = None,    **kwargs,) -> argparse.ArgumentParser:    parser = subparsers.add_parser(name, **kwargs)    add_common_options(parser, include_defaults=False)    if func is not None:        parser.set_defaults(func=func)    return parser  def build_parser() -> argparse.ArgumentParser:    parser = argparse.ArgumentParser(        prog="zed-eval",        description="Launch, monitor, and fetch remote SWE-Atlas agent evals on Modal.",        formatter_class=argparse.RawDescriptionHelpFormatter,        epilog=(            "Common workflows:\n"            "  zed-eval run swe-atlas -m sonnet-4.6       # launch the SWE-Atlas suite\n"            "  zed-eval run rf -m sonnet-4.6              # launch one benchmark\n"            "  zed-eval runs                              # what did I launch recently?\n"            "  zed-eval status                            # check my most recent run\n"            "  zed-eval status <run-id>                   # check a specific run\n"            "  zed-eval logs <run-id>                     # print controller logs\n"            "  zed-eval report <run-id> --fetch           # fetch + score a run\n"            "\n"            "After launching, a run id alone locates the run — namespace and\n"            "benchmark storage name are resolved from this machine's local run index."        ),    )    add_common_options(parser)    subparsers = parser.add_subparsers(dest="command", required=True)     doctor = add_command_parser(        subparsers,        "doctor",        func=command_doctor,        help="Check local prerequisites and defaults",    )    doctor.add_argument("--create-volume", action="store_true")     add_command_parser(        subparsers,        "deploy",        func=command_deploy,        help=(            "Publish harness code to Modal. This is the ONLY command that "            "deploys; deploying cancels in-flight runs. Run it once after "            "changing harness code — all other commands just invoke the "            "already-deployed functions and never deploy."        ),    )     build = add_command_parser(        subparsers,        "build",        func=command_build,        help="Build eval-cli on Modal into builds/<build-id>",    )    add_build_source_options(build)    build.add_argument(        "--build",        metavar="ID",        help="Reuse this build if it exists, otherwise create it with this id",    )    build.add_argument(        "--detach", action="store_true", help="Spawn the build and return immediately"    )     builds = add_command_parser(        subparsers,        "builds",        func=command_builds,        help="List content-addressed builds on the volume",    )    builds.add_argument("--details", action="store_true")    builds.add_argument("--json", action="store_true")    builds.add_argument("--limit", type=int, default=50)     run_cmd = add_command_parser(        subparsers,        "run",        func=command_run,        help="Run benchmarks — the everyday entry point",        description=(            "Launch runs by benchmark id, alias, or group. It picks the right "            "default judge for each benchmark and never prompts. The sibling "            "`swe-atlas` command launches SWE-Atlas parts (qna/rf/tw) with "            "interactive prompts."        ),

Lines 1–500 of 787. Load more