chore: update remote config (GPT-6.1 Terra, Kimi K3.1 Pro, Minimax M3.1 Flash, Mercury 3, thinking levels)

.agents/scripts/provider_utils.py

@@ -155,12 +155,16 @@ def get_api_key(env_var: str) -> str | None:
 # ---------------------------------------------------------------------------
 
 
+def fetch_openrouter_models() -> list[dict]:
+    """Returns OpenRouter's raw model records. No auth required."""
+    return fetch_json("https://openrouter.ai/api/v1/models")["data"]
+
+
 def fetch_openrouter() -> tuple[set[str], dict[str, str]]:
     """Returns (model_ids, expiring_dict) from OpenRouter. No auth required."""
-    data = fetch_json("https://openrouter.ai/api/v1/models")
     ids: set[str] = set()
     expiring: dict[str, str] = {}
-    for m in data["data"]:
+    for m in fetch_openrouter_models():
         ids.add(m["id"])
         if m.get("expiration_date"):
             expiring[m["id"]] = m["expiration_date"]
@@ -262,7 +266,7 @@ def fetch_vertex_with_aliases(project_id: str) -> set[str]:
     return with_aliases
 
 
-def _cloudflare_property(model: dict, property_id: str) -> Any:
+def cloudflare_property(model: dict, property_id: str) -> Any:
     for prop in model.get("properties") or []:
         if prop.get("property_id") == property_id:
             return prop.get("value")
@@ -331,7 +335,7 @@ def cloudflare_availability(
         if task.get("name") != CLOUDFLARE_TEXT_GENERATION_TASK:
             continue
         name = model["name"]
-        planned = _cloudflare_property(model, "planned_deprecation_date")
+        planned = cloudflare_property(model, "planned_deprecation_date")
         if planned:
             deprecation_dates[name] = str(planned)
             try:
@@ -343,19 +347,23 @@ def cloudflare_availability(
     return available, deprecation_dates
 
 
-def fetch_cloudflare(
-    api_key: str | None, account_id: str | None
-) -> tuple[set[str], dict[str, str]]:
-    """Returns (model_ids, deprecation_dates) for Cloudflare Workers AI.
+def fetch_cloudflare_models(api_key: str | None, account_id: str | None) -> list[dict]:
+    """Returns Cloudflare's raw model records, with their properties.
 
     Uses the authenticated models-search API when both credentials are set,
     otherwise the public catalog JSON. Never use LiteLLM's Cloudflare catalog:
     it is months stale.
     """
     if api_key and account_id:
-        models = _fetch_cloudflare_search(api_key, account_id)
-    else:
-        models = _fetch_cloudflare_public()
+        return _fetch_cloudflare_search(api_key, account_id)
+    return _fetch_cloudflare_public()
+
+
+def fetch_cloudflare(
+    api_key: str | None, account_id: str | None
+) -> tuple[set[str], dict[str, str]]:
+    """Returns (model_ids, deprecation_dates) for Cloudflare Workers AI."""
+    models = fetch_cloudflare_models(api_key, account_id)
     return cloudflare_availability(models, date.today())
 
 

.agents/skills/claude-maintain-models/SKILL.md

@@ -78,6 +78,8 @@ Run this check on **every invocation** of the skill, regardless of whether you'r
 
 3. **If a lagging provider now supports a recently-added model that isn't yet in its `KilnModel` entry**, flag it to the user and propose either bundling the provider addition into the current change or opening a separate PR. Do not silently add it.
 
+4. **Run the Cloudflare thinking-level checker** (see [Cloudflare thinking levels](#cloudflare-thinking-levels)). It checks every Cloudflare entry, not only the recent ones, because Cloudflare edits the effort lists of models Kiln already ships. Treat each ❌ finding like a backfill: fix the entry in the current change or propose a separate PR.
+
 ---
 
 ## Phase 2 – Gather Context
@@ -455,6 +457,7 @@ Replace the `` `Insert AI summary of PR using .agents/skills/open-pr/SKILL.md` `
 - [ ] Flags inherited from predecessor and adjusted for quirks
 - [ ] `reasoning_capable` defaulted to `False` for adaptive-reasoning models (only `True` for always-emits-reasoning models — see [Reasoning Capable Default](#reasoning-capable-default))
 - [ ] Thinking levels configured if model supports reasoning effort (see [Thinking Levels Reference](#thinking-levels-reference))
+- [ ] Cloudflare and OpenRouter: `check_thinking_levels.py` reports no ❌ finding (see [Cloudflare thinking levels](#cloudflare-thinking-levels) and [OpenRouter (general)](#openrouter-general))
 - [ ] Every `suggested_for_evals` entry with thinking levels defaults to `"medium"` (or `"low"`), never `"none"` (see [Judge Models Must Reason by Default](#judge-models-must-reason-by-default))
 - [ ] Preserve existing comments from predecessor (e.g. reasoning notes, MIME type groupings)
 - [ ] Zero-sum applied if model is suggested for evals/data gen
@@ -567,6 +570,15 @@ model.
   parametrized cases
 - `.agents/scripts/provider_utils.py` — add the provider's model-catalog
   endpoint so agent tooling can enumerate its models
+- Thinking levels — find out how the provider takes reasoning effort, and
+  where it publishes the values each model accepts. Add a quirks section
+  that names that source and says thinking levels are required wherever
+  it lists values. If the provider silently remaps unsupported values,
+  add the provider to `check_thinking_levels.py`, because a
+  paid test cannot see a remap.
+- `model_adapters/test_thinking_level_paid.py` — add the provider to the
+  provider list of `test_thinking_level_reasoning_content`, so each of its
+  thinking levels gets a paid test case
 
 **Known gap:** the generated Copilot API client
 (`app/desktop/studio_server/api_client/`) mirrors the remote Copilot service's
@@ -608,10 +620,16 @@ plumbing PR must not skip them.
 
 ### OpenRouter (general)
 - Slugs: `vendor/model-name`
+- Never add a `:free` slug. Free tiers have low rate limits and are temporary. If OpenRouter serves a model only as `:free`, wait for a paid slug.
 - Reasoning models: may need `require_openrouter_reasoning=True`
 - Some models: `openrouter_skip_required_parameters=True`
 - Logprobs: `logprobs_openrouter_options=True` if supported
 - Always `multimodal_requires_pdf_as_image=True` (OpenRouter's PDF routing breaks LiteLLM)
+- Thinking levels: offer only values in the model's `reasoning.supported_efforts` (see [Thinking Levels Reference](#thinking-levels-reference)). OpenRouter maps an unlisted effort to a listed one, so an extra level silently runs a different effort. When the model has no `reasoning.supported_efforts`, OpenRouter turns the effort into a thinking-token budget for Claude, so `CLAUDE_OPENROUTER_THINKING_LEVELS` (with Minimal) fits only those Claude models. `check_thinking_levels.py` checks every OpenRouter entry; a missing level list on OpenRouter is a note, because Kiln leaves levels off some models on purpose.
+
+### Fireworks AI and Together AI: thinking levels
+
+Neither publishes the efforts each model accepts, and both can accept an unsupported effort without an error. Fireworks takes `xhigh`, `max` and `minimal` on GLM 5.3 with HTTP 200. Together took a made-up effort on Nemotron 3.5, but Muse Spark Open's chat template rejects one. A paid test cannot see this. Add thinking levels on these providers only when the vendor documents the levels for that model (the model card or its chat template), and name that source in the PR. OpenAI and Anthropic return a 400 that names the supported levels, so their paid per-level test catches a wrong level.
 
 ### Featherless AI
 
@@ -651,10 +669,25 @@ Runs through LiteLLM's generic `openai/` route against Cloudflare's OpenAI-compa
   | `function_calling: "true"` | leave `supports_function_calling` at its default; set `False` if the tool tests fail |
   | `vision: "true"` | `supports_vision`, `multimodal_capable`, MIME types TXT/MD/JPG/PNG; start broad, narrow on 400s. Don't set `supports_doc_extraction`: the document extractor doesn't send the AI Gateway header, so extraction would bypass the user's gateway |
   | `reasoning: "true"` | keep `reasoning_capable=False` (see [Reasoning Capable Default](#reasoning-capable-default)); LiteLLM already reads `reasoning_content` |
-  | `reasoning_effort.supported_efforts` | source for `available_thinking_levels` if you add them; `mandatory: true` means reasoning cannot be turned off. Cloudflare edits these, so re-read on every run |
+  | `reasoning_effort.supported_efforts` present | **required:** set `available_thinking_levels` to exactly these values, and `default_thinking_level` to `default_effort`. See [Cloudflare thinking levels](#cloudflare-thinking-levels) |
+  | `reasoning_effort` without `supported_efforts` | no thinking levels; the catalog does not say which values work |
   | `require_workers_paid: "true"` | no Kiln field. Free-plan users get 403 / code 5035. Mention it in the PR |
   | `planned_deprecation_date` present | do **not** add the model |
 
+- <a id="cloudflare-thinking-levels"></a>**Thinking levels come from the catalog, and only from the catalog.** Kiln sends the level as `reasoning_effort`. For each model, `reasoning_effort` in the catalog lists `supported_efforts`, a `default_effort`, and `normalizes_to`: the values that Cloudflare silently changes to another effort. A remapped value returns HTTP 200, so a live call or a paid test cannot find it. Example: GLM 5.3 maps `medium` to `max`, so a "Medium" option runs the most expensive effort.
+  - Offer only values in `supported_efforts`, and each of them unless a completed paid test shows that the level does not work. The list is not always right: on DeepSeek V4 Pro 0813, DeepSeek V4 Flash 0731 and GLM 5.2, `none` still returns reasoning, so those entries have no Off level. A timeout, a rate limit or a credential error proves nothing about the level: keep the level and report it as untested. Do not copy the levels of the same model on another provider.
+  - Labels: `none` → `"Off"`, `low` → `"Low"`, `medium` → `"Medium"`, `high` → `"High"`, `xhigh` → `"Extra High"`, `max` → `"Max"`. List them from least to most effort.
+  - `default_thinking_level` is `default_effort`, unless the entry is suggested for evals (see [Judge Models Must Reason by Default](#judge-models-must-reason-by-default)).
+  - `mandatory: true` means `none` is not in the list; do not add an Off level.
+  - Reuse a constant when the values match exactly; else name a new one `{MODEL}_CLOUDFLARE_THINKING_LEVELS`.
+  - Cloudflare edits these lists. After every change to a Cloudflare entry, and on every run of this skill, run the checker below. It must report no ❌ finding:
+
+    ```bash
+    uv run python .agents/skills/claude-maintain-models/scripts/check_thinking_levels.py
+    ```
+
+    It compares every Cloudflare and OpenRouter entry in this checkout with the provider's catalog, and exits 1 on a level or default outside `supported_efforts`. For Cloudflare it also exits 1 on a missing level list, or levels on a model whose catalog lists none. `--provider cloudflare` or `--provider openrouter` checks one provider. A `note:` line is information only.
+  - Then run the paid `test_thinking_level_reasoning_content` for each new level. If `-k "<model_enum>"` collects no `cloudflare/...` case, `cloudflare` is missing from the provider list in `libs/core/kiln_ai/adapters/model_adapters/test_thinking_level_paid.py`; add it.
 - **Structured output:** these models take the OpenAI `response_format` envelope, so start from `json_schema` and fall back only on test failures. Cloudflare's JSON mode doesn't support streaming.
 - **Don't send `max_tokens`.** Live tests showed no truncation on the OpenAI-format models without it.
 - **Workers Paid:** many of the newer models need the Workers Paid plan. Paid tests need an account on it.
@@ -672,7 +705,7 @@ Runs through LiteLLM's generic `openai/` route against Cloudflare's OpenAI-compa
 
 ## Thinking Levels Reference
 
-No API provides the available thinking levels programmatically — they must be manually sourced. Use this lookup chain in priority order:
+Two providers publish the thinking levels per model, and their list is the only source for their entries: Cloudflare (see [Cloudflare thinking levels](#cloudflare-thinking-levels)) and OpenRouter (`reasoning.supported_efforts` in its models API, see [OpenRouter (general)](#openrouter-general)). For every other provider the levels must be sourced by hand. Use this lookup chain in priority order.
 
 ### Lookup Chain
 
@@ -688,11 +721,11 @@ No API provides the available thinking levels programmatically — they must be
 
 3. **Inherit from predecessor** — if the same family/tier model has a `_THINKING_LEVELS` dict, the new model very likely uses the same or a superset.
 
-4. **OpenRouter `supported_parameters`** — check if `reasoning` is present:
+4. **OpenRouter `reasoning`** — most models list their efforts:
    ```bash
-   curl -s https://openrouter.ai/api/v1/models | jq '.data[] | select(.id == "SLUG") | .supported_parameters'
+   curl -s https://openrouter.ai/api/v1/models | jq '.data[] | select(.id == "SLUG") | {reasoning, supported_parameters}'
    ```
-   If `reasoning` is absent, the model does not support effort levels — skip thinking levels entirely.
+   `reasoning.supported_efforts` lists the efforts (highest first) and `reasoning.default_effort` the default. If `reasoning` is absent and `supported_parameters` has no `reasoning`, the model does not support effort levels — skip thinking levels entirely. OpenRouter documents that it maps an unsupported effort to the nearest supported one for some models, so an extra level may silently run a different effort.
 
 5. **Smoke test** — as a last resort, send a request with an invalid effort level and check the error message, which often enumerates the valid values.
 
@@ -732,7 +765,6 @@ Reuse when levels match exactly. Create a new constant only if levels differ. Th
 ### Sources That Do NOT Work
 
 These were investigated and confirmed to lack thinking level data:
-- **OpenRouter API** — only boolean `reasoning` in `supported_parameters`
 - **LiteLLM catalog** — only `supports_reasoning: true/false`
 - **Google Gemini models API** — only `thinking: true/false`
 - **OpenAI `/v1/models` endpoint** — minimal object with no capability fields