lib/cli/tests/test_model_catalog.py - main - githubmirror/crewai

lib/cli/tests/test_model_catalog.py
606 строк22 KB
João Moura
fix(cli): key model-catalog cache by exact API key, shorten TTL, skip Ollama (#6468)
08 июл 2026, 04:47
08 июл 2026, 04:47835b93d
"""Tests for the dynamic model catalog used by the crew-creation wizard."""

from __future__ import annotations

import json
import time

import pytest

import crewai_cli.model_catalog as mc

_ALL_KEY_ENVS = [
    "OPENAI_API_KEY",
    "ANTHROPIC_API_KEY",
    "GEMINI_API_KEY",
    "GOOGLE_API_KEY",
    "GROQ_API_KEY",
    "CEREBRAS_API_KEY",
    "OLLAMA_API_BASE",
    "API_BASE",
    "OLLAMA_HOST",
]

FALLBACK_ANTHROPIC = [
    ("claude-opus-4-6", "Claude Opus 4.6"),
    ("claude-sonnet-4-6", "Claude Sonnet 4.6"),
]


@pytest.fixture(autouse=True)
def isolated_env(monkeypatch, tmp_path):
    """Point the cache at a temp dir and clear provider keys for every test."""
    monkeypatch.setattr(mc, "_cache_dir", lambda: tmp_path)
    mc._reset_litellm_memo()  # clear the process-level LiteLLM memo per test
    for key in _ALL_KEY_ENVS:
        monkeypatch.delenv(key, raising=False)


# ── version / label helpers ──────────────────────────────────────


def test_version_key_parses_embedded_date():
    date_int, version = mc._version_key("claude-3-5-sonnet-20241022")
    assert date_int == 20241022
    assert version == (3, 5)


def test_version_key_parses_dashed_date():
    date_int, _ = mc._version_key("gpt-4o-2024-08-06")
    assert date_int == 20240806


def test_version_key_version_only():
    date_int, version = mc._version_key("claude-opus-4-6")
    assert date_int == 0
    assert version == (4, 6)


def test_version_key_ranks_newer_higher():
    older = mc._version_key("claude-sonnet-4-5")
    newer = mc._version_key("claude-sonnet-4-6")
    assert newer > older


def test_is_chat_model_rejects_non_chat():
    assert mc._is_chat_model("gpt-4.1-mini")
    assert not mc._is_chat_model("text-embedding-3-large")
    assert not mc._is_chat_model("whisper-1")
    assert not mc._is_chat_model("dall-e-3")


def test_search_substring_not_treated_as_non_chat():
    # 'search' must not drop legitimate completion models: a token like
    # *-search-preview, or 'research' (which contains 'search' as a substring).
    assert mc._is_chat_model("gpt-4o-search-preview")
    assert mc._is_chat_model("o3-deep-research")
    # genuine non-chat markers still filter
    assert not mc._is_chat_model("text-embedding-3-large")


def test_humanize():
    assert mc._humanize("gpt-4.1-mini") == "GPT 4.1 Mini"
    assert mc._humanize("anthropic/claude-opus-4-6") == "Claude Opus 4 6"
    # size suffixes uppercased, acronyms/brands cased, o-series preserved, ':' split
    assert mc._humanize("openai/gpt-oss-120b") == "GPT OSS 120B"
    assert mc._humanize("qwen/qwen3-32b") == "Qwen3 32B"
    assert mc._humanize("deepseek-r1-distill-llama-70b") == "DeepSeek R1 Distill Llama 70B"
    assert mc._humanize("o3-mini") == "o3 Mini"
    assert mc._humanize("chatgpt-4o-latest") == "ChatGPT 4o Latest"
    assert mc._humanize("llama3.3:70b") == "Llama3.3 70B"
    assert mc._humanize("gemma2-9b-it") == "Gemma2 9B IT"


# ── vendor tier ──────────────────────────────────────────────────


def test_vendor_anthropic_ranks_by_date_and_uses_display_name(monkeypatch):
    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
    payload = {
        "data": [
            {
                "id": "claude-3-5-sonnet-20240620",
                "display_name": "Claude 3.5 Sonnet (old)",
                "created_at": "2024-06-20T00:00:00Z",
            },
            {
                "id": "claude-opus-4-6",
                "display_name": "Claude Opus 4.6",
                "created_at": "2026-02-01T00:00:00Z",
            },
            {
                "id": "claude-haiku-4-5-20251001",
                "display_name": "Claude Haiku 4.5",
                "created_at": "2025-10-01T00:00:00Z",
            },
        ]
    }
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: payload)

    models = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)

    # Newest first by created_at, display names preserved.
    assert models[0] == ("claude-opus-4-6", "Claude Opus 4.6")
    assert models[1] == ("claude-haiku-4-5-20251001", "Claude Haiku 4.5")
    assert models[2] == ("claude-3-5-sonnet-20240620", "Claude 3.5 Sonnet (old)")


def test_vendor_openai_filters_non_chat_models(monkeypatch):
    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
    payload = {
        "data": [
            {"id": "gpt-4.1", "created": 1_700_000_000},
            {"id": "text-embedding-3-large", "created": 1_800_000_000},
            {"id": "whisper-1", "created": 1_800_000_000},
            {"id": "gpt-5.5", "created": 1_750_000_000},
        ]
    }
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: payload)

    models = mc.get_provider_models("openai", [])
    ids = [m for m, _ in models]

    assert ids == ["gpt-5.5", "gpt-4.1"]  # embeddings/whisper dropped, newest first


def test_vendor_gemini_requires_generate_content(monkeypatch):
    monkeypatch.setenv("GEMINI_API_KEY", "key")
    payload = {
        "models": [
            {
                "name": "models/gemini-2.5-pro",
                "displayName": "Gemini 2.5 Pro",
                "supportedGenerationMethods": ["generateContent"],
            },
            {
                "name": "models/text-embedding-004",
                "displayName": "Embedding",
                "supportedGenerationMethods": ["embedContent"],
            },
            {
                "name": "models/gemini-1.5-pro",
                "displayName": "Gemini 1.5 Pro",
                "supportedGenerationMethods": ["generateContent"],
            },
        ]
    }
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: payload)

    models = mc.get_provider_models("gemini", [])
    ids = [m for m, _ in models]

    # "models/" prefix stripped, embedding excluded, newer version first.
    assert ids == ["gemini-2.5-pro", "gemini-1.5-pro"]


def test_openai_excludes_fine_tunes_and_checkpoints(monkeypatch):
    # Fine-tunes/checkpoints have recent `created` timestamps and would otherwise
    # crowd out (and rank above) the base models — they must be excluded so the
    # picker shows clean foundation models.
    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
    payload = {
        "data": [
            {"id": "ft:gpt-4o-mini-2024-07-18:crewai::DyJG86uF", "created": 1_900_000_000},
            {
                "id": "ft:gpt-4o-mini-2024-07-18:crewai::DyJG7Q9N:ckpt-step-84",
                "created": 1_900_000_001,
            },
            {"id": "gpt-5.5", "created": 1_800_000_000},
            {"id": "gpt-4.1", "created": 1_700_000_000},
        ]
    }
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: payload)

    ids = [m for m, _ in mc.get_provider_models("openai", [])]
    assert ids == ["gpt-5.5", "gpt-4.1"]  # fine-tunes + checkpoints dropped


def test_vendor_gemini_paginates(monkeypatch):
    monkeypatch.setenv("GEMINI_API_KEY", "key")
    pages = {
        None: {
            "models": [
                {
                    "name": "models/gemini-3.5-flash",
                    "displayName": "Gemini 3.5 Flash",
                    "supportedGenerationMethods": ["generateContent"],
                }
            ],
            "nextPageToken": "p2",
        },
        "p2": {
            "models": [
                {
                    "name": "models/gemini-2.5-pro",
                    "displayName": "Gemini 2.5 Pro",
                    "supportedGenerationMethods": ["generateContent"],
                }
            ]
        },
    }

    def fetch(url, headers=None, params=None):
        return pages[(params or {}).get("pageToken")]

    monkeypatch.setattr(mc, "_http_get_json", fetch)

    ids = sorted(m for m, _ in mc.get_provider_models("gemini", []))
    # Both pages contributed (newest-first ranking is _finalize's job).
    assert ids == ["gemini-2.5-pro", "gemini-3.5-flash"]


def test_vendor_gemini_first_page_error_uses_fallback(monkeypatch):
    # A total (first-page) Gemini failure with a key set must fall back to the
    # curated list, not be mistaken for a successful empty result.
    monkeypatch.setenv("GEMINI_API_KEY", "key")

    def boom(*a, **k):
        raise RuntimeError("gemini down")

    monkeypatch.setattr(mc, "_http_get_json", boom)
    models = mc.get_provider_models("gemini", [("gemini-x", "Gemini X")])
    assert models == [("gemini-x", "Gemini X")]


def test_vendor_gemini_keeps_partial_on_later_page_error(monkeypatch):
    monkeypatch.setenv("GEMINI_API_KEY", "key")

    def fetch(url, headers=None, params=None):
        if (params or {}).get("pageToken"):
            raise RuntimeError("page 2 down")
        return {
            "models": [
                {
                    "name": "models/gemini-3.5-flash",
                    "displayName": "Gemini 3.5 Flash",
                    "supportedGenerationMethods": ["generateContent"],
                }
            ],
            "nextPageToken": "p2",
        }

    monkeypatch.setattr(mc, "_http_get_json", fetch)

    # Page-1 models are kept; the later-page error doesn't force the fallback.
    models = mc.get_provider_models("gemini", [("fallback-x", "Fallback X")])
    assert [m for m, _ in models] == ["gemini-3.5-flash"]


def test_ollama_empty_response_not_filled_with_fallback(monkeypatch):
    # A reachable Ollama with nothing installed -> empty (manual entry), not the
    # curated suggestions the crew can't actually run.
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: {"models": []})
    assert mc.get_provider_models("ollama", [("llama3.3", "Llama 3.3")]) == []


def test_ollama_unreachable_uses_fallback(monkeypatch):
    # Server down (fetch raises) is different from empty -> fall back to suggestions.
    def boom(*a, **k):
        raise RuntimeError("connection refused")

    monkeypatch.setattr(mc, "_http_get_json", boom)
    models = mc.get_provider_models("ollama", [("llama3.3", "Llama 3.3")])
    assert models == [("llama3.3", "Llama 3.3")]


def test_ollama_excludes_embedding_models(monkeypatch):
    # /api/tags lists everything installed, including embeddings — filter them.
    monkeypatch.setattr(
        mc,
        "_http_get_json",
        lambda *a, **k: {
            "models": [
                {"model": "llama3.3:70b"},
                {"model": "nomic-embed-text"},
                {"model": "mxbai-embed-large"},
            ]
        },
    )
    ids = [m for m, _ in mc.get_provider_models("ollama", [])]
    assert ids == ["llama3.3:70b"]


def test_ollama_base_honors_ollama_host(monkeypatch):
    # OLLAMA_HOST (scheme-less runtime convention) is resolved with a scheme.
    monkeypatch.setenv("OLLAMA_HOST", "10.0.0.5:11434")
    assert mc._ollama_base() == "http://10.0.0.5:11434"


def test_ollama_recovery_not_blocked_by_negative_cache(monkeypatch):
    # Ollama down -> fallback, but not negatively cached; once the server is up
    # the next call fetches live models rather than serving suggestions.
    calls = {"n": 0}

    def flaky(*a, **k):
        calls["n"] += 1
        if calls["n"] == 1:
            raise RuntimeError("connection refused")
        return {"models": [{"model": "llama-installed"}]}

    monkeypatch.setattr(mc, "_http_get_json", flaky)
    first = mc.get_provider_models("ollama", [("llama3.3", "Llama 3.3")])
    assert first == [("llama3.3", "Llama 3.3")]  # down -> fallback (not cached)
    second = mc.get_provider_models("ollama", [("llama3.3", "Llama 3.3")])
    assert [m for m, _ in second] == ["llama-installed"]  # recovered live


def test_gemini_honors_google_api_key(monkeypatch):
    # GOOGLE_API_KEY (equivalent to GEMINI_API_KEY in crewai) enables the live tier.
    monkeypatch.setenv("GOOGLE_API_KEY", "key")
    monkeypatch.setattr(
        mc,
        "_http_get_json",
        lambda *a, **k: {
            "models": [
                {
                    "name": "models/gemini-3.5-flash",
                    "displayName": "Gemini 3.5 Flash",
                    "supportedGenerationMethods": ["generateContent"],
                }
            ]
        },
    )
    models = mc.get_provider_models("gemini", [("gemini-x", "Gemini X")])
    assert [m for m, _ in models] == ["gemini-3.5-flash"]  # live, not fallback


def test_curated_label_overrides_raw_vendor_label(monkeypatch):
    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
    payload = {"data": [{"id": "gpt-5.5", "created": 1}]}
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: payload)

    models = mc.get_provider_models("openai", [("gpt-5.5", "GPT-5.5 (curated)")])
    assert models == [("gpt-5.5", "GPT-5.5 (curated)")]


def test_truncates_to_max_models(monkeypatch):
    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
    payload = {
        "data": [{"id": f"gpt-test-{i}", "created": i} for i in range(20)]
    }
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: payload)

    models = mc.get_provider_models("openai", [])
    assert len(models) == mc.MAX_MODELS


# ── litellm tier ─────────────────────────────────────────────────


def test_litellm_tier_for_uncurated_provider(monkeypatch):
    # A provider with no curated fallback ([]) -> the LiteLLM feed is consulted.
    litellm_data = {
        "claude-opus-4-6": {"litellm_provider": "anthropic", "mode": "chat"},
        "claude-sonnet-4-5": {"litellm_provider": "anthropic", "mode": "chat"},
        "voyage-embed": {"litellm_provider": "anthropic", "mode": "embedding"},
        "gpt-4.1": {"litellm_provider": "openai", "mode": "chat"},
    }
    mc._litellm_cache_file().write_text(json.dumps(litellm_data), encoding="utf-8")

    models = mc.get_provider_models("anthropic", [])  # empty == uncurated
    ids = [m for m, _ in models]

    # Only anthropic chat models, embedding + other providers excluded.
    assert ids == ["claude-opus-4-6", "claude-sonnet-4-5"]


def test_null_litellm_provider_does_not_crash(monkeypatch):
    # A present-but-null litellm_provider must be skipped, not raise.
    litellm_data = {
        "weird-model": {"litellm_provider": None, "mode": "chat"},
        "anthropic.claude-v2": {"litellm_provider": "bedrock", "mode": "chat"},
    }
    mc._litellm_cache_file().write_text(json.dumps(litellm_data), encoding="utf-8")

    models = mc.get_provider_models("bedrock", [])
    assert [m for m, _ in models] == ["anthropic.claude-v2"]


def test_litellm_strips_provider_prefix(monkeypatch):
    litellm_data = {
        "gemini/gemini-1.5-pro": {"litellm_provider": "gemini", "mode": "chat"},
    }
    mc._litellm_cache_file().write_text(json.dumps(litellm_data), encoding="utf-8")

    models = mc.get_provider_models("gemini", [])
    assert models == [("gemini-1.5-pro", "Gemini 1.5 Pro")]


# ── fallback + caching ───────────────────────────────────────────


def test_falls_back_when_everything_fails(monkeypatch):
    # No key, no litellm cache, network raises -> curated fallback verbatim.
    def boom(*a, **k):
        raise RuntimeError("network down")

    monkeypatch.setattr(mc, "_http_get_json", boom)
    models = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)
    assert models == FALLBACK_ANTHROPIC


def test_result_is_cached(monkeypatch):
    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
    calls = {"n": 0}

    def fetch(*a, **k):
        calls["n"] += 1
        return {"data": [{"id": "claude-opus-4-6", "created_at": "2026-01-01T00:00:00Z"}]}

    monkeypatch.setattr(mc, "_http_get_json", fetch)

    first = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)
    # Second call must hit the cache and not touch the network again.
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: pytest.fail("refetched"))
    second = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)

    assert first == second
    assert calls["n"] == 1


def test_curated_fallback_preferred_over_litellm(monkeypatch):
    # The feed lags real releases, so a non-empty curated fallback must win even
    # when a fresh LiteLLM cache is present (regression: Anthropic's feed lacked
    # Fable 5 / Opus 4.8 / Sonnet 5).
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: pytest.fail("no net"))
    litellm_data = {
        "claude-opus-4-6": {"litellm_provider": "anthropic", "mode": "chat"},
    }
    mc._litellm_cache_file().write_text(json.dumps(litellm_data), encoding="utf-8")

    models = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)
    assert models == FALLBACK_ANTHROPIC


def test_added_key_bypasses_negative_cache(monkeypatch):
    # A no-key call negatively-caches the fallback; adding a key afterwards must
    # fetch live models rather than serve the cached fallback (distinct cache key).
    first = mc.get_provider_models("openai", [("gpt-x", "GPT X")])
    assert first == [("gpt-x", "GPT X")]  # no key -> fallback

    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
    monkeypatch.setattr(
        mc, "_http_get_json", lambda *a, **k: {"data": [{"id": "gpt-5.5", "created": 1}]}
    )
    second = mc.get_provider_models("openai", [("gpt-x", "GPT X")])
    assert [m for m, _ in second] == ["gpt-5.5"]  # live fetch, not cached fallback


def test_invalid_litellm_cache_falls_through_to_download(monkeypatch):
    # A corrupt-but-fresh cache must neither crash the picker nor block a
    # recoverable download — it falls through and refetches.
    mc._litellm_cache_file().write_text("[1, 2, 3]", encoding="utf-8")
    monkeypatch.setattr(
        mc,
        "_http_get_json",
        lambda *a, **k: {
            "anthropic.claude-v2": {"litellm_provider": "bedrock", "mode": "chat"}
        },
    )
    models = mc.get_provider_models("bedrock", [])
    assert [m for m, _ in models] == ["anthropic.claude-v2"]  # recovered via download


def test_litellm_fetch_attempted_once_per_process(monkeypatch):
    # With no cache and a failing download, the feed is fetched at most once per
    # process — repeated lookups (across providers) must not re-hit the network.
    calls = {"n": 0}

    def boom(*a, **k):
        calls["n"] += 1
        raise RuntimeError("offline")

    monkeypatch.setattr(mc, "_http_get_json", boom)
    mc.get_provider_models("bedrock", [])
    mc.get_provider_models("azure", [])
    assert calls["n"] == 1  # memoized after the first failed attempt


def test_litellm_fills_uncurated_bedrock(monkeypatch):
    # No vendor fetcher and no curated fallback -> LiteLLM feed fills the gap.
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: pytest.fail("no net"))
    litellm_data = {
        "anthropic.claude-v2": {"litellm_provider": "bedrock", "mode": "chat"},
    }
    mc._litellm_cache_file().write_text(json.dumps(litellm_data), encoding="utf-8")

    models = mc.get_provider_models("bedrock", [])
    assert models == [("anthropic.claude-v2", "Anthropic.claude V2")]


def test_failed_fetch_is_negatively_cached(monkeypatch):
    # A failed vendor fetch must not be retried on every call — the fallback is
    # cached briefly so the picker doesn't re-hit the timeout-prone endpoint.
    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
    calls = {"n": 0}

    def boom(*a, **k):
        calls["n"] += 1
        raise RuntimeError("down")

    monkeypatch.setattr(mc, "_http_get_json", boom)
    first = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)
    second = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)

    assert first == second == FALLBACK_ANTHROPIC
    assert calls["n"] == 1  # second call served from the negative cache


def test_bad_cache_json_does_not_crash(monkeypatch):
    # A corrupt cache whose root is not a mapping must not raise (get_provider_models
    # is documented to never raise).
    mc._catalog_cache_file().write_text("[1, 2, 3]", encoding="utf-8")

    models = mc.get_provider_models("anthropic", FALLBACK_ANTHROPIC)
    assert models == FALLBACK_ANTHROPIC


def test_ollama_is_not_cached_reflects_installed_changes(monkeypatch):
    # Ollama is local and never cached: the picker re-probes /api/tags on every
    # call, so a model deleted locally drops out immediately (no stale entry).
    responses = iter(
        [
            {"models": [{"model": "llama3.3"}, {"model": "qwen3"}]},
            {"models": [{"model": "llama3.3"}]},  # qwen3 deleted between calls
        ]
    )
    monkeypatch.setattr(mc, "_http_get_json", lambda *a, **k: next(responses))

    first = {m for m, _ in mc.get_provider_models("ollama", [])}
    second = {m for m, _ in mc.get_provider_models("ollama", [])}

    assert first == {"llama3.3", "qwen3"}
    assert second == {"llama3.3"}  # re-probed, not served from a stale cache


def test_ollama_never_written_to_catalog_cache(monkeypatch):
    monkeypatch.setattr(
        mc, "_http_get_json", lambda *a, **k: {"models": [{"model": "llama3.3"}]}
    )
    mc.get_provider_models("ollama", [])
    assert not mc._catalog_cache_file().exists()


def test_different_api_key_uses_separate_cache_entry(monkeypatch):
    # Cache is keyed by the exact key: switching keys must refetch, not serve
    # the previous account's cached list.
    monkeypatch.setenv("OPENAI_API_KEY", "sk-account-A")
    monkeypatch.setattr(
        mc, "_http_get_json", lambda *a, **k: {"data": [{"id": "gpt-5.5", "created": 1}]}
    )
    assert [m for m, _ in mc.get_provider_models("openai", [])] == ["gpt-5.5"]

    monkeypatch.setenv("OPENAI_API_KEY", "sk-account-B")
    monkeypatch.setattr(
        mc, "_http_get_json", lambda *a, **k: {"data": [{"id": "gpt-4.1", "created": 1}]}
    )
    # New key -> distinct cache entry -> refetch, not the account-A cache.
    assert [m for m, _ in mc.get_provider_models("openai", [])] == ["gpt-4.1"]


def test_cache_key_hashes_key_and_never_stores_it(monkeypatch):
    monkeypatch.setenv("OPENAI_API_KEY", "sk-super-secret")
    key = mc._cache_key("openai")
    assert key.startswith("openai#") and key != "openai#nokey"
    assert "sk-super-secret" not in key  # only a digest, never the raw key


def test_dynamic_cache_expires_after_catalog_ttl(monkeypatch):
    # A dynamic entry older than the (now short) catalog TTL is not served.
    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
    entry_key = mc._cache_key("anthropic")
    stale_ts = time.time() - (mc._CATALOG_TTL + 5)
    mc._catalog_cache_file().write_text(
        json.dumps(
            {
                entry_key: {
                    "ts": stale_ts,
                    "source": "dynamic",
                    "models": [["stale-model", "Stale"]],
                }
            }
        ),
        encoding="utf-8",
    )

    assert mc._read_catalog_cache("anthropic") is None