{ "repository": "BerriAI/litellm", "ref": "bd8eeedfc2f0638c26b017007853592cf71287a4", "methods": [ { "slug": "formal", "title": "Formal verification and model checking" }, { "slug": "static_analysis", "title": "Static analysis" }, { "slug": "testing", "title": "Testing" }, { "slug": "formatting", "title": "Formatting and hygiene" }, { "slug": "supply_chain", "title": "Supply chain" }, { "slug": "unclassified", "title": "Unclassified" } ], "verifications": [ { "tool": "cargo-check", "method": "static_analysis", "commands": [ "cargo check -p litellm-python-bridge --locked --no-default-features --features abi3${features:+,$features}" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-test.steps[11] (Test token counter feature combinations)" } ], "upstream": "https://github.com/rust-lang/cargo", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "clippy", "method": "static_analysis", "commands": [ "cargo clippy --workspace --all-targets --locked -- -D warnings" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-lint.steps[5]" }, { "path": "litellm-rust/clippy.toml", "locator": "config file" } ], "upstream": "https://github.com/rust-lang/rust-clippy", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "codeql", "method": "static_analysis", "commands": [ "uses github/codeql-action/analyze" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/codeql.yml", "locator": "jobs.analyze.steps[2] (Perform CodeQL Analysis)" } ], "upstream": "https://github.com/github/codeql", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "eslint", "method": "static_analysis", "commands": [ "eslint --no-warn-ignored --pass-on-unpruned-suppressions ${eslint_files[@]}", "eslint . -f json -o $RUNNER_TEMP/lint-report.json" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ui.steps[4] (Lint changed files (prettier + eslint))" }, { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ui.steps[5] (Check lint budgets)" }, { "path": "ui/litellm-dashboard/eslint.config.mjs", "locator": "config file" } ], "upstream": "https://github.com/eslint/eslint", "docs": "https://eslint.org/docs/latest/use/command-line-interface#exit-codes", "exit_codes": { "0": "no errors", "1": "linting errors found", "2": "configuration problem or internal error" }, "autofix": true, "declared_only": false }, { "tool": "go-vet", "method": "static_analysis", "commands": [ "go vet ./..." ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-terraform-provider.yml", "locator": "jobs.provider-checks.steps[3] (go vet)" } ], "upstream": "https://github.com/golang/go", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "pyright", "method": "static_analysis", "commands": [ "basedpyright tests/e2e tests/e2e_harness", "basedpyright --pythonpath .venv/bin/python --project tests/typing/pyrightconfig.json" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[23] (Check tests/e2e basedpyright (zero errors))" }, { "path": ".github/workflows/test-unit.yml", "locator": "jobs.openai-sdk-compat.steps[8] (Check consumer types with the installed SDK)" }, { "path": "pyproject.toml", "locator": "[dependency-groups] dev: basedpyright" }, { "path": "pyrightconfig.json", "locator": "config file" }, { "path": "tests/pyrightconfig.json", "locator": "config file" }, { "path": "tests/typing/pyrightconfig.json", "locator": "config file" } ], "upstream": "https://github.com/microsoft/pyright", "docs": "https://microsoft.github.io/pyright/#/command-line", "exit_codes": { "0": "no errors", "1": "errors reported", "2": "fatal error", "3": "config file could not be read or parsed", "4": "illegal command-line parameters" }, "autofix": false, "declared_only": false }, { "tool": "ruff-check", "method": "static_analysis", "commands": [ "ruff check .", "ruff check --config ruff-tests.toml tests" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[15] (Run Ruff linting)" }, { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[16] (Run Ruff linting (test tree))" }, { "path": "pyproject.toml", "locator": "[dependency-groups] dev: ruff" }, { "path": "ruff.toml", "locator": "config file" } ], "upstream": "https://github.com/astral-sh/ruff", "docs": "https://docs.astral.sh/ruff/linter/#exit-codes", "exit_codes": { "0": "no violations", "1": "violations found", "2": "invalid configuration, CLI options or internal error" }, "autofix": true, "declared_only": false }, { "tool": "semgrep", "method": "static_analysis", "commands": [ "semgrep scan --config .semgrep/rules . --error" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.semgrep.steps[3] (Run Semgrep (custom rules))" } ], "upstream": "https://github.com/semgrep/semgrep", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "tsc", "method": "static_analysis", "commands": [ "tsc --noEmit" ], "triggers": [ "ci", "local" ], "evidence": [ { "path": ".github/workflows/test-vscode-extension.yml", "locator": "jobs.vscode-extension.steps[3] (Typecheck)" }, { "path": "vscode-extension/package.json", "locator": "scripts.typecheck" }, { "path": "vscode-extension/package.json", "locator": "scripts.vscode:prepublish" }, { "path": "vscode-extension/package.json", "locator": "devDependencies: typescript" } ], "upstream": "https://github.com/microsoft/TypeScript", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "mypy", "method": "static_analysis", "commands": [], "triggers": [], "evidence": [ { "path": "pyproject.toml", "locator": "[dependency-groups] dev: mypy" } ], "upstream": "https://github.com/python/mypy", "docs": "https://mypy.readthedocs.io/en/stable/command_line.html", "exit_codes": { "0": "no type errors", "1": "type errors found", "2": "crash or invalid invocation" }, "autofix": false, "declared_only": true }, { "tool": "pylint", "method": "static_analysis", "commands": [], "triggers": [], "evidence": [ { "path": "pyproject.toml", "locator": "[dependency-groups] ci: pylint" } ], "upstream": "https://github.com/pylint-dev/pylint", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": true }, { "tool": "cargo-test", "method": "testing", "commands": [ "cargo test -p litellm-db --locked -- --include-ignored", "cargo nextest run --workspace --locked", "cargo test --workspace --doc --locked", "cargo test -p litellm-token-counter --locked --no-default-features --features $features", "cargo test -p litellm-auth-gcp --locked --no-default-features" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-rust-db.yml", "locator": "jobs.db-queries.steps[10] (Test database queries)" }, { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-test.steps[9]" }, { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-test.steps[10]" }, { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-test.steps[11] (Test token counter feature combinations)" }, { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-test.steps[12] (Test secret manager feature combinations)" } ], "upstream": "https://github.com/rust-lang/cargo", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "ci-matrix", "method": "testing", "commands": [ "analyze (include: 3)", "runtime-image (dockerfile: 2)", "unit (include: 41)", "openai-sdk-compat (openai-version: 3)", "redis-compat (redis-py: 4)" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/codeql.yml", "locator": "jobs.analyze.strategy.matrix" }, { "path": ".github/workflows/image-scan.yml", "locator": "jobs.runtime-image.strategy.matrix" }, { "path": ".github/workflows/test-unit.yml", "locator": "jobs.unit.strategy.matrix" }, { "path": ".github/workflows/test-unit.yml", "locator": "jobs.openai-sdk-compat.strategy.matrix" }, { "path": ".github/workflows/test-unit.yml", "locator": "jobs.redis-compat.strategy.matrix" } ], "upstream": "https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/run-job-variations", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "go-test", "method": "testing", "commands": [ "go test -timeout 120s ./..." ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-terraform-provider.yml", "locator": "jobs.provider-checks.steps[5] (Test)" } ], "upstream": "https://github.com/golang/go", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "hypothesis", "method": "testing", "commands": [], "triggers": [], "evidence": [ { "path": "pyproject.toml", "locator": "[dependency-groups] dev: hypothesis" } ], "upstream": "https://github.com/HypothesisWorks/hypothesis", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "mutmut", "method": "testing", "commands": [ "mutmut run 2>&1" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/mutation-test.yml", "locator": "jobs.mutation.steps[11] (Run mutmut)" }, { "path": "pyproject.toml", "locator": "[tool.mutmut]" } ], "upstream": "https://github.com/boxed/mutmut", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "proptest", "method": "testing", "commands": [], "triggers": [], "evidence": [ { "path": "litellm-rust/Cargo.toml", "locator": "[workspace.dependencies]: proptest" } ], "upstream": "https://github.com/proptest-rs/proptest", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "pytest", "method": "testing", "commands": [ "pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA", "pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v", "pytest --noconftest tests/integration/database/image/test_offline_image_migration.py -v", "pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v", "pytest --noconftest tests/integration/database/image/test_ui_image_serves_offline.py -v" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/e2e_record_replay.yml", "locator": "jobs.record.steps[8] (Record the replayable e2e lane)" }, { "path": ".github/workflows/e2e_record_replay.yml", "locator": "jobs.replay.steps[10] (Replay the replayable e2e lane)" }, { "path": ".github/workflows/image-scan.yml", "locator": "jobs.image-scan.steps[5] (Verify offline migration as a non-root uid)" }, { "path": ".github/workflows/image-scan.yml", "locator": "jobs.runtime-image.steps[4] (Verify offline migration as a non-root uid)" }, { "path": ".github/workflows/image-scan.yml", "locator": "jobs.migrations-image.steps[3] (Verify offline migration as a non-root uid)" }, { "path": ".github/workflows/image-scan.yml", "locator": "jobs.gateway-image.steps[3] (Verify the gateway serves offline as a non-root uid)" }, { "path": ".github/workflows/image-scan.yml", "locator": "jobs.ui-image.steps[3] (Verify the UI serves offline as an arbitrary uid with a read-only root fs)" }, { "path": ".github/workflows/image-scan.yml", "locator": "jobs.backend-image.steps[4] (Verify the backend serves offline as a non-root uid)" }, { "path": ".github/workflows/interactions-spec-canary.yml", "locator": "jobs.interactions-spec-canary.steps[5] (Run the compliance tests against Google's live spec)" }, { "path": ".github/workflows/test-e2e-redis-chaos.yml", "locator": "jobs.redis-chaos-e2e.steps[8] (Run the Redis chaos load test)" }, { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[24] (Run the e2e harness tests)" }, { "path": ".github/workflows/test-linting.yml", "locator": "jobs.secret-scan.steps[3] (Run secret scan test)" } ], "upstream": "https://github.com/pytest-dev/pytest", "docs": "https://docs.pytest.org/en/stable/reference/exit-codes.html", "exit_codes": { "0": "all tests passed", "1": "some tests failed", "2": "interrupted by the user", "3": "internal error", "4": "command line usage error", "5": "no tests were collected" }, "autofix": false, "declared_only": false }, { "tool": "vitest", "method": "testing", "commands": [ "vitest run" ], "triggers": [ "ci", "local" ], "evidence": [ { "path": ".github/workflows/test-vscode-extension.yml", "locator": "jobs.vscode-extension.steps[4] (Unit tests)" }, { "path": "vscode-extension/package.json", "locator": "scripts.test" }, { "path": "vscode-extension/package.json", "locator": "devDependencies: vitest" }, { "path": "ui/litellm-dashboard/vitest.config.ts", "locator": "config file" }, { "path": "vscode-extension/vitest.config.mts", "locator": "config file" } ], "upstream": "https://github.com/vitest-dev/vitest", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": "jest", "method": "testing", "commands": [], "triggers": [], "evidence": [ { "path": "package.json", "locator": "devDependencies: jest" } ], "upstream": "https://github.com/jestjs/jest", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": true }, { "tool": "playwright", "method": "testing", "commands": [], "triggers": [], "evidence": [ { "path": "tests/e2e/ui/playwright.config.ts", "locator": "config file" } ], "upstream": "https://github.com/microsoft/playwright", "docs": null, "exit_codes": {}, "autofix": false, "declared_only": true }, { "tool": "prettier", "method": "formatting", "commands": [ "prettier --check ${prettier_files[@]}" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ui.steps[4] (Lint changed files (prettier + eslint))" }, { "path": "ui/litellm-dashboard/.prettierrc", "locator": "config file" } ], "upstream": "https://github.com/prettier/prettier", "docs": "https://prettier.io/docs/cli#exit-codes", "exit_codes": { "0": "everything formatted", "1": "something was not formatted (--check)", "2": "prettier error" }, "autofix": true, "declared_only": false }, { "tool": "rustfmt", "method": "formatting", "commands": [ "cargo fmt --all --check" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-lint.steps[2]" } ], "upstream": "https://github.com/rust-lang/rustfmt", "docs": null, "exit_codes": {}, "autofix": true, "declared_only": false }, { "tool": "uv-lock", "method": "supply_chain", "commands": [ "uv lock --check" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[7] (Check uv.lock is up to date)" }, { "path": ".github/workflows/test-mcp-dependency-resolution.yml", "locator": "jobs.resolve.steps[5] (Verify lockfile)" } ], "upstream": "https://github.com/astral-sh/uv", "docs": null, "exit_codes": {}, "autofix": true, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "Bearer ${LITELLM_MASTER_KEY} --header 'Content-Type: application/json' --data '{model:gpt-4,messages:[{role:system,content:You are a helpful assistant.},{role:user,content:Hello!}]}' ${base_url}/chat/completions)" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-merge-smoke.yml", "locator": "jobs.proxy-build-from-pip.steps[8] (Run the startup tests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "bash $GITHUB_WORKSPACE/.github/scripts/select_ui_test_scope.sh)" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-unit.yml", "locator": "jobs.ui-unit.steps[6] (Run UI unit tests (Vitest))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "bun test scripts/auto-close-duplicates.test.ts" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/auto-close-duplicates.yml", "locator": "jobs.sweep-tests.steps[2] (Test the sweep)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "bun test scripts/classify-issue.test.ts scripts/label-issue.test.ts" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/issue_classifier.yml", "locator": "jobs.classify-issue-tests.steps[2] (Test the gate, the validation and the label step)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "bun test scripts/comment-fixed-issue.test.ts" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/issue_fixed_comment.yml", "locator": "jobs.comment-fixed-issue-tests.steps[2] (Test the closer lookup, the release placement, the comment and the superseded pull request close)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "bun test scripts/flag-duplicate-issue.test.ts" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/duplicate_issue_check.yml", "locator": "jobs.flag-tests.steps[2] (Test the flag step)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "bun test scripts/sync-issue-labels.test.ts" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/issue_label_sync.yml", "locator": "jobs.sync-issue-labels-tests.steps[2] (Test the sync)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "check_deps base/enterprise/pyproject.toml pr/enterprise/pyproject.toml enterprise/pyproject.toml" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/guard-fork-dependencies.yml", "locator": "jobs.guard.steps[3] (Reject new dependencies in pyproject.toml files)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "check_deps base/litellm-proxy-extras/pyproject.toml pr/litellm-proxy-extras/pyproject.toml litellm-proxy-extras/pyproject.toml" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/guard-fork-dependencies.yml", "locator": "jobs.guard.steps[3] (Reject new dependencies in pyproject.toml files)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "check_deps base/pyproject.toml pr/pyproject.toml pyproject.toml" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/guard-fork-dependencies.yml", "locator": "jobs.guard.steps[3] (Reject new dependencies in pyproject.toml files)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "coverage run --rcfile=/dev/null --branch --include=*/scripts/build_core_distribution.py scripts/build_core_distribution.py --out-dir dist/core" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-mcp-dependency-resolution.yml", "locator": "jobs.core-distribution.steps[4] (Build the core wheel and source distribution)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "coverage xml --rcfile=/dev/null -o coverage-core.xml" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-mcp-dependency-resolution.yml", "locator": "jobs.core-distribution.steps[6] (Verify isolated core installations)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "diff $$base_ref --unified=0 --no-color -- *.py)" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-changed (line 150)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "diff --name-only $$base_ref -- *.py)" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-ruff-FULL-dev (line 156)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "diff --name-only --diff-filter=ACMR --merge-base $$base_ref -- litellm/*.py)" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-check-changed (line 137)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "do eslint_files+=($f)" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ui.steps[4] (Lint changed files (prettier + eslint))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "do file=$${range%%:*}" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-changed (line 150)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "done" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-changed (line 150)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "done < $RUNNER_TEMP/eslint_files.txt" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ui.steps[4] (Lint changed files (prettier + eslint))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "else echo $$files" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-check-changed (line 137)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "else echo No changed .py files to check." ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-ruff-FULL-dev (line 156)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "for spec in data.get(project, {}).get(dependencies, []):" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/guard-fork-dependencies.yml", "locator": "jobs.guard.steps[3] (Reject new dependencies in pyproject.toml files)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "for spec in group:" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/guard-fork-dependencies.yml", "locator": "jobs.guard.steps[3] (Reject new dependencies in pyproject.toml files)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "ggshield secret scan repo ." ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.secret-scan.steps[4] (Run ggshield secret scan)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "go run ./tools/endpointaudit -provider-dir ./litellm -spec ${RUNNER_TEMP}/openapi.json -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-terraform-provider.yml", "locator": "jobs.endpoint-drift.steps[11] (Audit provider endpoints against the schema)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.8.2" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target install-helm-unittest (line 103)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "helm plugin list" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target install-helm-unittest (line 103)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "helm unittest -f tests/*.yaml $chart" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-unit.yml", "locator": "jobs.helm.steps[4] (Run unit tests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "helm unittest -f tests/*.yaml helm/litellm-helm" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target test-unit-helm (line 284)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "m = pep508.match(spec)" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/guard-fork-dependencies.yml", "locator": "jobs.guard.steps[3] (Reject new dependencies in pyproject.toml files)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "node scripts/check-lint-budgets.mjs $RUNNER_TEMP/lint-report.json eslint-budgets.json" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ui.steps[5] (Check lint budgets)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-integration.yml", "locator": "jobs.integration.steps[9] (Install browser dependencies and build the dashboard)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-integration.yml", "locator": "jobs.integration.steps[9] (Install browser dependencies and build the dashboard)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "nohup .venv/bin/python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 > ${RUNNER_TEMP}/fake-openai.log 2>&1 &" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-merge-smoke.yml", "locator": "jobs.proxy-build-from-pip.steps[5] (Start the fake OpenAI endpoint)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "nothing to check.\"" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ui.steps[1] (Collect changed files)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "perl -ne ' if (/^diff --git a\\/(.*) b\\//) { $$file = $$1" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-changed (line 150)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "print $$file:$$start:1-$$end:999\\n" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-changed (line 150)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ../tests/documentation_tests/test_circular_imports.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[25] (Check for circular imports)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[37] (ban_copy_deepcopy_kwargs)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/callback_manager_test.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[28] (callback_manager_test)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[40] (check_e2e_no_raw_requests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_fastuuid_usage.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[38] (check_fastuuid_usage)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[26] (check_get_model_cost_key_performance)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[23] (check_guardrail_apply_decorator)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_licenses.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[7] (check_licenses)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_migrations_lock_timeout.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[42] (check_migrations_lock_timeout)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[41] (check_migrations_no_data_rewrites)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_no_none_string_error_fields.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[43] (check_no_none_string_error_fields)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[45] (check_no_publicly_known_master_key)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_prisma_binary_cache.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[11] (check_prisma_binary_cache)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_provider_folders_documented.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[8] (check_provider_folders_documented)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_py310_typing_imports.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[39] (check_py310_typing_imports)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_telemetry_imports.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[36] (check_telemetry_imports)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_unbounded_in_lists.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[47] (check_unbounded_in_lists (fails on findings not in the baseline))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[35] (check_unsafe_enterprise_import)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_workflow_job_name_collisions.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[13] (check_workflow_job_name_collisions)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/check_workflow_startup_safety.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[12] (check_workflow_startup_safety)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/code_qa_check_tests.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[25] (code_qa_check_tests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/enforce_llms_folder_style.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[33] (enforce_llms_folder_style)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/ensure_async_clients_test.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[32] (ensure_async_clients_test)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/info_log_check.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[22] (info_log_check)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/litellm_logging_code_coverage.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[31] (litellm_logging_code_coverage)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/memory_test.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[48] (memory_test)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[34] (prevent_key_leaks_in_exceptions)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/recursive_detector.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[29] (recursive_detector)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/router_code_coverage.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[19] (router_code_coverage)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/router_surface_check.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[20] (router_surface_check)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/test_ban_set_verbose.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[24] (test_ban_set_verbose)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/test_chat_completion_imports.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[21] (test_chat_completion_imports)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/test_proxy_types_import.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[27] (test_proxy_types_import)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/code_coverage_tests/test_router_strategy_async.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[30] (test_router_strategy_async)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/documentation_tests/test_api_docs.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[51] (documentation_test_api_docs)" }, { "path": ".github/workflows/test-unit.yml", "locator": "jobs.docs.steps[11] (Run documentation validation tests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/documentation_tests/test_circular_imports.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-unit.yml", "locator": "jobs.docs.steps[11] (Run documentation validation tests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/documentation_tests/test_env_keys.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[49] (documentation_test_env_keys)" }, { "path": ".github/workflows/test-unit.yml", "locator": "jobs.docs.steps[11] (Run documentation validation tests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python ./tests/documentation_tests/test_router_settings.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[50] (documentation_test_router_settings)" }, { "path": ".github/workflows/test-unit.yml", "locator": "jobs.docs.steps[11] (Run documentation validation tests)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/assert_ci_coverage.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ci-coverage.steps[2] (Assert every test file and Dockerfile is invoked by a job)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/assert_ci_coverage.py --shards" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-unit.yml", "locator": "jobs.assert-shard-coverage.steps[1] (Assert every test directory and file is claimed by a shard)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/assert_ci_coverage.py --slices" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.ci-coverage.steps[3] (Assert no -k expression deselects a file from every job that globs it)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/assert_secret_absent.py test-results" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-integration.yml", "locator": "jobs.integration.steps[13] (Check results for the license)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file ${RUNNER_TEMP}/egress-hits.jsonl" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/e2e_record_replay.yml", "locator": "jobs.replay.steps[11] (Stop the egress sentinel and assert zero provider egress)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/e2e_egress_sentinel.py serve --host api.openai.com --host api.anthropic.com --hits-file ${RUNNER_TEMP}/egress-hits.jsonl --ready-file ${RUNNER_TEMP}/egress-ready --pid-file ${RUNNER_TEMP}/egress.pid > ${RUNNER_TEMP}/egress-sentinel.log 2>&1 &" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/e2e_record_replay.yml", "locator": "jobs.replay.steps[9] (Start the egress sentinel)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/run_llm_translation_tests.py" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target test-llm-translation (line 288)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/scan_keywords.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/issue-keyword-labeler.yml", "locator": "jobs.scan-and-label.steps[1] (Scan for provider keywords)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-wheel.steps[11]" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python .github/scripts/verify_linux_native_wheel.py dist/*.whl" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-wheel.steps[7]" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/check_mcp_operation_boundary.py" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[14] (Check MCP operation boundary)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/check_schema_compatibility.py --base-ref $SCHEMA_BASE_REF" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-unit.yml", "locator": "jobs.unit.steps[6] (Install dependencies and check schema)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/generate_provider_endpoints_support.py --check" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.code-quality.steps[9] (check_provider_endpoints_support_generated)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/openapi_docs_gate.py --base $GATE_BASE_SHA" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[20] (Check OpenAPI docs (POST routes without a request schema, description or 200 schema, delta vs merge-base counts))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/test_quality_gate.py --base $GATE_BASE_SHA" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[19] (Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/type_check_gate.py --base $GATE_BASE_SHA" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[22] (Check basedpyright (delta vs merge-base counts))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/type_check_gate.py --emit-counts-dir $RUNNER_TEMP/lint-counts" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/publish-lint-base-counts.yml", "locator": "jobs.publish.steps[8] (Emit basedpyright counts for HEAD)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python scripts/type_discipline_gate.py --base $GATE_BASE_SHA" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-linting.yml", "locator": "jobs.python.steps[18] (Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-wheel.steps[8]" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "skipping unit tests.\"" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-unit.yml", "locator": "jobs.ui-unit.steps[6] (Run UI unit tests (Vitest))" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "terraform fmt -recursive -check -diff" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-terraform-modules.yml", "locator": "jobs.module.steps[2] (fmt)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "terraform test" ], "triggers": [ "ci" ], "evidence": [ { "path": ".github/workflows/test-terraform-modules.yml", "locator": "jobs.module.steps[5] (test)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "trap rm -rf \"$$temporary\" EXIT HUP INT TERM" ], "triggers": [ "ci", "local" ], "evidence": [ { "path": ".github/workflows/test-rust.yml", "locator": "jobs.rust-wheel.steps[9] (Run pytest tests/test_litellm_rust with the compiled extension)" }, { "path": "Makefile", "locator": "target test-rust-extension (line 234)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "true" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target install-helm-unittest (line 103)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "use make check\" >&2" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target pre-commit (line 229)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "while read range" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-changed (line 150)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "xargs $(UV_RUN) ruff check" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-ruff-FULL-dev (line 156)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false }, { "tool": null, "method": "unclassified", "commands": [ "xargs $(UV_RUN) ruff format --check --exclude /enterprise/" ], "triggers": [ "local" ], "evidence": [ { "path": "Makefile", "locator": "target lint-format-check-changed (line 137)" } ], "upstream": null, "docs": null, "exit_codes": {}, "autofix": false, "declared_only": false } ] }