"""Dotted Bedrock/Vertex model-label normalization (split out of PR #6607). ``us.anthropic.claude-opus-5`` carries a cross-region routing prefix and a vendor namespace. Left intact both survive into the human label, so the turn footer rendered "Us.anthropic.claude Opus 5". The normalization is implemented twice — inlined in ``_get_label_for_model()`` (api/config.py) and as ``_stripDottedModelPrefix()`` (static/ui.js) — so these tests are PAIRED: one table drives both sides, and a divergence fails. The allow-list is closed on purpose. A generic "drop leading letters-only dot segments" loop rewrote arbitrary uncatalogued IDs (``deepseek.v3`` → "V3", ``foo.bar.baz`` → "BAZ"); those cases are pinned below. """ from __future__ import annotations import json import shutil import subprocess import sys from pathlib import Path import pytest REPO_ROOT = Path(__file__).parent.parent.resolve() if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) from api.config import _get_label_for_model # noqa: E402 UI_JS = (REPO_ROOT / "static" / "ui.js").read_text(encoding="utf-8") def _derive_set_from_config(marker: str) -> set[str]: """Read an allow-list out of PRODUCTION source instead of retyping it. A retyped copy silently stops matching when the real set grows -- which is exactly how ``global`` and then ``luma``/``twelvelabs``/``ibm`` shipped mislabeled. Same technique as test_every_catalog_dotted_id_loses_its_routing_prefix. """ import re as _re config_src = (REPO_ROOT / "api" / "config.py").read_text(encoding="utf-8") start = config_src.index(marker) body = config_src[start:config_src.index("}", start)] found = {m.lower() for m in _re.findall(r'"([a-z0-9-]+)"', body)} assert found, f"could not derive {marker!r} from api/config.py" return found _PRODUCTION_REGIONS = _derive_set_from_config("_regions = {") # (model_id, expected_normalized_id) — what the dotted-prefix step must leave # behind, BEFORE the shared cosmetic title-casing that both sides apply after. STRIP_CASES = [ # --- documented Bedrock/Vertex shapes: prefix is plumbing -------------- ("us.anthropic.claude-opus-5", "claude-opus-5"), ("eu.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-sonnet-4-5-20250929-v1"), ("apac.anthropic.claude-haiku-4", "claude-haiku-4"), # ``global`` is a routing head the catalog actually ships (six # ``global.anthropic.claude-*`` IDs at api/config.py:1901-1909, and the # first-party routing notes use it as the canonical Bedrock shape). Omitting # it from the region set left every one of those labels reading # "Global.anthropic.claude Opus 4 7". ("global.anthropic.claude-opus-4-7", "claude-opus-4-7"), ("global.anthropic.claude-opus-4-6-v1", "claude-opus-4-6-v1"), ("global.anthropic.claude-sonnet-4-6", "claude-sonnet-4-6"), ("global.anthropic.claude-opus-4-5-20251101-v1:0", "claude-opus-4-5-20251101-v1"), ("global.anthropic.claude-sonnet-4-5-20250929-v1:0", "claude-sonnet-4-5-20250929-v1"), ("global.anthropic.claude-haiku-4-5-20251001-v1:0", "claude-haiku-4-5-20251001-v1"), ("us-gov.anthropic.claude-opus-5", "claude-opus-5"), ("mistral.mistral-large-2407-v1:0", "mistral-large-2407-v1"), ("amazon.nova-pro-v1:0", "nova-pro-v1"), ("meta.llama3-70b-instruct-v1:0", "llama3-70b-instruct-v1"), # --- must be left BYTE-INTACT ------------------------------------------ # Vendor is the whole name: stripping it would render the model as "V3". ("deepseek.v3", "deepseek.v3"), # Uncatalogued vendor: not our shape, do not touch. ("foo.bar.baz", "foo.bar.baz"), ("acme.super-model-9", "acme.super-model-9"), # A known region head with an unknown vendor is not our shape either. ("us.foo.bar", "us.foo.bar"), # Version dots must survive. ("gpt-4.1", "gpt-4.1"), ("qwen3.6-35b", "qwen3.6-35b"), ("o1.5-preview", "o1.5-preview"), # URI-scheme IDs are paths, not dotted namespaces. ("https://host/v1.2/model", "https://host/v1.2/model"), # No dot at all. ("claude-opus-5", "claude-opus-5"), ] # End-to-end labels through the real backend function. LABEL_CASES = [ ("us.anthropic.claude-opus-5", "Claude Opus 5"), ("mistral.mistral-large-2407-v1:0", "Mistral Large 2407 V1"), ("gpt-4.1", "GPT 4.1"), ("qwen3.6-35b", "Qwen3.6 35B"), ] @pytest.mark.parametrize("model_id,expected", LABEL_CASES) def test_backend_label_drops_dotted_plumbing(model_id, expected): assert _get_label_for_model(model_id, []) == expected @pytest.mark.parametrize("model_id", [ "deepseek.v3", "foo.bar.baz", "acme.super-model-9", "us.foo.bar", ]) def test_backend_label_keeps_uncatalogued_vendor_name(model_id): """The vendor word must not be silently deleted from an unknown ID.""" vendor = model_id.split(".")[0] label = _get_label_for_model(model_id, []) assert vendor.lower() in label.lower(), ( f"{model_id!r} lost its vendor: label={label!r}" ) def _js_strip(model_ids: list[str]) -> list[str]: node = shutil.which("node") if not node: pytest.skip("node not available") def extract(src: str, name: str) -> str: start = src.index(f"function {name}(") depth = 0 started = False for i in range(start, len(src)): if src[i] == "{": depth += 1 started = True elif src[i] == "}": depth -= 1 if started and depth == 0: return src[start:i + 1] raise AssertionError(f"unbalanced braces extracting {name}") # The two Sets the helper closes over are declared immediately above it. sets_start = UI_JS.index("const _BEDROCK_REGION_PREFIXES") sets_end = UI_JS.index("function _stripDottedModelPrefix") script = "\n".join([ UI_JS[sets_start:sets_end], extract(UI_JS, "_stripDottedModelPrefix"), "const ids = JSON.parse(process.argv[1]);", "console.log(JSON.stringify(ids.map(_stripDottedModelPrefix)));", ]) proc = subprocess.run( [node, "--input-type=module", "-e", script, json.dumps(model_ids)], capture_output=True, text=True, timeout=30, check=True, ) return json.loads(proc.stdout) def test_frontend_strip_matches_the_table(): """The JS half must normalize exactly as specified — same table.""" ids = [c[0] for c in STRIP_CASES] js = _js_strip(ids) for (model_id, expected), js_out in zip(STRIP_CASES, js, strict=True): assert js_out == expected, ( f"js drifted for {model_id!r}: js={js_out!r} expected={expected!r}" ) def _js_model_labels(model_ids: list[str]) -> dict[str, str]: """Drive the REAL ``getModelLabel()`` from static/ui.js under Node. ``_js_strip()`` above exercises only ``_stripDottedModelPrefix()``. That is the wrong unit for a parity claim: ``getModelLabel()`` applies a whole retry chain AFTER the strip (``_dynamicModelLabels`` → ``STATIC_LABELS`` → ``'anthropic/' + id`` → a ``claude-`` prettifier). A test that stops at the strip cannot observe any of it. Only SINKS are stubbed (``_dynamicModelLabels`` starts empty, exactly as it is before a catalog fetch; ``_fmtOllamaLabel`` is identity). Every decision function is the shipped source, eval'd — so drift fails here instead of rendering wrong in the picker. """ node = shutil.which("node") if not node: pytest.skip("node not available") proc = subprocess.run( [node, "-e", _GET_MODEL_LABEL_DRIVER, str(REPO_ROOT / "static" / "ui.js"), json.dumps(model_ids)], capture_output=True, text=True, timeout=30, ) assert proc.returncode == 0, f"node driver failed: {proc.stderr}" return json.loads(proc.stdout) # Reuses the boundary-slicing approach already proven in # tests/test_issue3429_uri_scheme_model_label.py: regex literals inside # getModelLabel() defeat a naive brace counter, so bound it by the next # top-level function instead. _GET_MODEL_LABEL_DRIVER = r""" const fs = require('fs'); const ui = fs.readFileSync(process.argv[1], 'utf8'); const start = ui.indexOf('function getModelLabel('); if (start < 0) throw new Error('getModelLabel not found'); const after = ui.indexOf('\nfunction _gatewayProviderName(', start); if (after < 0) throw new Error('getModelLabel end boundary not found'); // Sinks only: no catalog has been fetched yet, so dynamic labels are empty. const _dynamicModelLabels = {}; function _fmtOllamaLabel(s){ return s; } // The dotted normalizer and the two Sets it closes over sit just above // getModelLabel(); without them the eval'd function ReferenceErrors. const _stripStart = ui.indexOf('const _BEDROCK_REGION_PREFIXES'); if (_stripStart < 0 || _stripStart > start) throw new Error('strip block not found'); eval(ui.slice(_stripStart, start)); eval(ui.slice(start, after)); const out = {}; for (const m of JSON.parse(process.argv[2])) out[m] = getModelLabel(m); process.stdout.write(JSON.stringify(out)); """ def test_frontend_and_backend_agree_that_no_namespace_survives(): """The real contract, driven through the real ``getModelLabel()``. NOT string equality of the two labels. The frontend and the backend are deliberately different formatters and always have been: on the unmodified base commit ``claude-opus-5`` (no dot, untouched by this change) already labels as ``'claude-opus-5'`` in JS and ``'Claude Opus 5'`` in Python, and ``openai/gpt-4o`` as ``'GPT-4o'`` vs ``'GPT 4O'``. Asserting equality would fail on pre-existing behavior this PR does not touch. The reason that divergence is not a bug: ``getModelLabel()`` consults ``_dynamicModelLabels`` FIRST (static/ui.js:6991), and that map is populated from the server's own ``label`` field (static/ui.js:3483, 3494, 3606). Once a catalog is loaded the backend label wins in the UI. The JS formatter is the pre-fetch fallback, so its job is to be no WORSE than the backend, not byte-identical to it. What must hold on both sides — the actual defect this PR fixes — is that no routing/vendor namespace leaks into a human label. """ ids = [c[0] for c in STRIP_CASES] js_labels = _js_model_labels(ids) for model_id, expected_stripped in STRIP_CASES: js_label = js_labels[model_id] py_label = _get_label_for_model(model_id, []) # A namespace head is only "leaked" if it was supposed to be stripped. # Cases pinned as byte-intact (deepseek.v3, foo.bar.baz, gpt-4.1) must # KEEP their dotted head, so they are checked in the opposite direction: # the head must still be present rather than silently deleted. if expected_stripped == model_id: if "." not in model_id: # No dot at all (``claude-opus-5``): the dotted-prefix step is # not involved. Nothing for this test to assert beyond the strip # being a no-op. assert _js_strip([model_id]) == [model_id], ( f"dotted strip must be a no-op for {model_id!r}" ) continue if "://" in model_id: # URI-scheme IDs are paths, not dotted namespaces. # getModelLabel() deliberately renders the last meaningful path # segment (the #3429 fix, pinned in # tests/test_issue3429_uri_scheme_model_label.py), so a full-URL # assertion here would contradict that test. What matters for # THIS PR is only that the dotted-prefix strip left it alone. assert _js_strip([model_id]) == [model_id], ( f"dotted strip must not touch the URI-scheme id {model_id!r}" ) continue head = model_id.split(".", 1)[0].lower() # Only a LETTERS-ONLY head is a namespace candidate. ``gpt-4.1``, # ``qwen3.6-35b`` and ``o1.5-preview`` carry a VERSION dot, so their # "head" (``gpt-4``) is not a namespace and title-casing legitimately # reshapes it ("GPT 4.1"). What must hold for those is that the dot # itself survives -- asserted directly below. if head.replace("-", "").isalpha(): assert head in js_label.lower(), ( f"{model_id!r} is pinned byte-intact but the frontend deleted " f"its head {head!r}: {js_label!r}" ) assert head in py_label.lower(), ( f"{model_id!r} is pinned byte-intact but the backend deleted " f"its head {head!r}: {py_label!r}" ) else: assert "." in js_label and "." in py_label, ( f"{model_id!r} lost its version dot: js={js_label!r} " f"py={py_label!r}" ) continue head = model_id.split(".", 1)[0].lower() for side, label in (("frontend", js_label), ("backend", py_label)): assert f"{head}." not in label.lower(), ( f"{side} leaked the routing namespace for {model_id!r}: {label!r}" ) # A REGION head must never survive as a leading word either # ("Us anthropic claude Opus 5"). This check does NOT apply to a # vendor head, because a vendor name legitimately reappears inside # some model names -- ``mistral.mistral-large-2407-v1:0`` correctly # labels as "Mistral Large 2407 V1". Same rule as # test_known_bedrock_vendors_are_all_covered below. if head in _PRODUCTION_REGIONS: assert not label.lower().startswith(f"{head} "), ( f"{side} kept the region head as a word for {model_id!r}: {label!r}" ) def test_frontend_retry_chain_reaches_the_static_label_table(): """Pin the behavior that makes the JS fallback WORTH having. This is the post-strip chain the previous test could not see. Removing the retry block at static/ui.js:7053-7071 leaves these rendering as raw IDs (``'claude-sonnet-4-5'`` instead of ``'Sonnet 4.5'``) — verified by excising the block and re-running this driver. """ cases = { # Lands on STATIC_LABELS['anthropic/claude-sonnet-4-5'] after the strip. "us.anthropic.claude-sonnet-4-5": "Sonnet 4.5", "eu.anthropic.claude-haiku-3-5": "Haiku 3.5", # No table entry: the claude- prettifier drops the vendor word, the # -YYYYMMDD date pin and the -v1 revision. "global.anthropic.claude-opus-4-5-20251101-v1:0": "Opus 4 5", "us-gov.anthropic.claude-opus-5": "Opus 5", } got = _js_model_labels(list(cases)) for model_id, expected in cases.items(): assert got[model_id] == expected, ( f"frontend label drift for {model_id!r}: {got[model_id]!r} != {expected!r}" ) for label in got.values(): assert "claude-" not in label.lower(), f"vendor word survived: {label!r}" def test_every_catalog_dotted_id_loses_its_routing_prefix(): """Catalog-driven guard against region/vendor-set drift. The allow-lists and the shipped catalog are two lists that must agree. They didn't, twice: first ``global`` was missing (six IDs mislabeled), then ``luma``/``twelvelabs``/``ibm`` were missing (real Bedrock vendors rendering as "Us.luma.ray 2"). An earlier version of this test scraped only three-segment ``..`` literals and therefore inspected **6** of the **75** dotted catalog IDs — reassuring, but nearly blind. This version: - scrapes ANY quoted ``id`` value (single or double quotes, any segment count); - derives the offending prefixes from the PRODUCTION allow-lists rather than a retyped copy, so a set that grows without test updates is still covered; - skips version dots (``qwen3.6-plus``, ``gpt-5.4``), which are not namespaces. """ import re as _re config_src = (REPO_ROOT / "api" / "config.py").read_text(encoding="utf-8") # Derive the real allow-lists out of production source, don't retype them. def _set_literal(marker: str) -> set[str]: start = config_src.index(marker) body = config_src[start:config_src.index("}", start)] return {m.lower() for m in _re.findall(r'"([a-z0-9-]+)"', body)} regions = _set_literal("_regions = {") vendors = _set_literal("_vendors = {") assert regions and vendors, "could not derive allow-lists from api/config.py" namespace_heads = regions | vendors ids = { i for i in _re.findall(r"""['"]id['"]\s*:\s*['"]([^'"]+)['"]""", config_src) if "." in i } assert len(ids) > 20, f"catalog scrape found only {len(ids)} dotted ids" offenders = [] for model_id in sorted(ids): head = model_id.split(".")[0].lower() # Only IDs whose head is a KNOWN namespace should be stripped; a version # dot such as `qwen3.6-plus` has no namespace head and is left alone. if head not in namespace_heads: continue label = _get_label_for_model(model_id, []) if head in label.lower().replace(" ", "."): offenders.append((model_id, label)) assert not offenders, ( "catalog IDs still carry a routing/vendor prefix in their label — add the " f"missing head to the region/vendor sets: {offenders}" ) def test_known_bedrock_vendors_are_all_covered(): """Real Bedrock foundation-model vendors must all be in the allow-list. These were shipping mislabeled: `luma.ray-2` rendered as "Luma.ray 2", `us.twelvelabs.marengo-embed-2-7` as "Us.twelvelabs.marengo Embed 2 7". The assertion is that no DOTTED NAMESPACE survives — not that the vendor word is absent, because a vendor legitimately reappears inside some model names (``mistral.mistral-large-2407`` → "Mistral Large 2407"). """ for model in [ "luma.ray-2", "twelvelabs.marengo-embed-2-7", "ibm.granite-3-8b-instruct", "anthropic.claude-opus-5", "amazon.nova-pro-v1:0", "mistral.mistral-large-2407-v1:0", ]: head = model.split(".")[0] label = _get_label_for_model(model, []) assert f"{head}." not in label.lower(), ( f"{model!r} kept its vendor namespace: {label!r}" ) # And with a region prefix in front. regional = f"us.{model}" rlabel = _get_label_for_model(regional, []) assert "us." not in rlabel.lower() and f"{head}." not in rlabel.lower(), ( f"{regional!r} kept its namespace: {rlabel!r}" ) def test_global_region_is_recognized_in_both_implementations(): """Explicit pin for the reported gap, both sides.""" label = _get_label_for_model("global.anthropic.claude-opus-4-7", []) assert "global" not in label.lower(), label assert "anthropic" not in label.lower(), label assert _js_strip(["global.anthropic.claude-opus-4-7"]) == [ "claude-opus-4-7" ] def test_frontend_helper_exists_and_is_used(): assert "function _stripDottedModelPrefix" in UI_JS assert "_stripDottedModelPrefix(_last)" in UI_JS # The generic letters-only loop must be gone. assert "/^[a-z]+$/i.test(_segs[_i]" not in UI_JS def test_backend_uses_a_closed_allow_list(): """Guard the design: the fix must not regress to a generic prefix loop.""" config_src = (REPO_ROOT / "api" / "config.py").read_text(encoding="utf-8") idx = config_src.index("def _get_label_for_model") block = config_src[idx:idx + 4000] assert "_regions" in block and "_vendors" in block, ( "the dotted-prefix strip must be gated on explicit provider allow-lists" ) assert "while _i < len(_segs) - 1 and _segs[_i].isalpha()" not in block, ( "the generic letters-only loop rewrites uncatalogued IDs" )