update_cddl.py

"""Regenerate ``common/webref_cddl.bzl``, the pins behind BiDi schema generation.

The WebDriver BiDi (and related) CDDL grammars are not published as an npm
package; they are extracted from the edited specs and committed to the
``ed/cddl`` directory of https://github.com/w3c/webref . We pin a single
webref commit plus a sha256 for the ``-all`` (union) CDDL file and the ``dfns``
definition index of each merged spec so the Bazel build fetches them
reproducibly. The per-end ``-local``/``-remote`` splits are skipped: generation
merges the union, so only the ``-all`` files are consumed.

Which specs are merged is not a list kept here. It is WebDriver BiDi itself plus
every spec linked from the "External specifications" section of the rendered
core spec (the specs that define WebDriver BiDi modules). Each link is resolved to
its webref shortname through webref's ``ed/index.json``, which also names the
spec's CDDL extracts. Getting a new module merged therefore means getting it
listed upstream, not editing this script.

The pins point at the tip of webref's "main" branch (the continuous reffy
extraction; the "curated" branch is a separate published lineage with no shared
history, so main keeps pin-to-pin diffs auditable) and at the tip of
w3c/webdriver-bidi's "gh-pages" branch (its rendered ``index.html`` carries the
prose section anchors that become the readable ``#type-``/``#command-`` spec
links, and that branch is a separate repo, so it is resolved separately).

The pins always advance to the two tips. The update-cddl workflow only opens a PR
when the regenerated ``common/bidi/schema.json`` changes, so a repin that changes
nothing the schema consumes is dropped there rather than filtered here.

Regenerates ``common/webref_cddl.bzl`` in full and the extension's ``use_repo(...)``
block in ``MODULE.bazel``.
"""

import hashlib
import json
import os
import re
from pathlib import Path

import urllib3

http = urllib3.PoolManager()
root_dir = Path(os.path.realpath(__file__)).parent.parent

WEBREF_REPO = "w3c/webref"
WEBREF_BRANCH = "main"
WEBREF_RAW = "https://raw.githubusercontent.com/w3c/webref/{commit}/ed/{path}"
CORE_SHORTNAME = "webdriver-bidi"

BIDI_SPEC_REPO = "w3c/webdriver-bidi"
BIDI_SPEC_BRANCH = "gh-pages"
BIDI_SPEC_FILE = "index.html"
BIDI_SPEC_REPO_NAME = "webdriver_bidi_spec_html"
BIDI_SPEC_RAW = "https://raw.githubusercontent.com/w3c/webdriver-bidi/{commit}/index.html"

API_HEADERS = {"Accept": "application/vnd.github+json", "User-Agent": "selenium-update-cddl"}

# Not pinned yet: Firefox's grammar lives in mozilla-central (remote/doc/webdriver-bidi) and is
# copied verbatim under common/bidi until D327393 lands there. Keyed by vendor namespace.
VENDOR_CDDL_FILES = {
    "moz": [
        "//common/bidi:Commands.cddl",
        "//common/bidi:Debugging.cddl",
        "//common/bidi:Fields.cddl",
        "//common/bidi:Profiler.cddl",
    ],
}

BZL_FILE = root_dir / "common" / "webref_cddl.bzl"
MODULE_FILE = root_dir / "MODULE.bazel"

BZL_TEMPLATE = '''# DO NOT EDIT! This file is generated by scripts/update_cddl.py.
# Regenerate with: `bazel run //scripts:update_cddl`
"""Pinned webref CDDL grammars, dfns indexes, and the rendered WebDriver BiDi spec that feed BiDi schema generation.

The merged specs are WebDriver BiDi and the specs its "External specifications" section lists;
see scripts/update_cddl.py for how they are resolved and when the pins advance.
"""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")

# The w3c/webref "main" commit every CDDL grammar and dfns index below is taken from.
_COMMIT = "{webref_commit}"
_CDDL_BASE_URL = "https://raw.githubusercontent.com/w3c/webref/{{commit}}/ed/cddl".format(commit = _COMMIT)
_DFNS_BASE_URL = "https://raw.githubusercontent.com/w3c/webref/{{commit}}/ed/dfns".format(commit = _COMMIT)

# The rendered WebDriver BiDi spec, pinned at a w3c/webdriver-bidi "gh-pages" commit. Only this
# file carries the readable prose section ids (`#type-`/`#command-`/`#event-`/`#module-`).
_BIDI_SPEC_HTML_COMMIT = "{bidi_commit}"
_BIDI_SPEC_HTML_SHA256 = "{bidi_sha256}"
_BIDI_SPEC_HTML_URL = "https://raw.githubusercontent.com/w3c/webdriver-bidi/{{commit}}/index.html".format(
    commit = _BIDI_SPEC_HTML_COMMIT,
)

# (repo_name, filename, sha256). Each grammar is downloaded as "spec.cddl" and each dfns
# index as "dfns.json", so they are referenced as @<repo_name>//file:spec.cddl and
# @<repo_name>//file:dfns.json.
_CDDL_FILES = [
{cddl_entries}
]

_DFNS_FILES = [
{dfns_entries}
]

# The merged specs as labels, for the BUILD files that feed schema generation: the core
# grammar, then the extension grammars, then one dfns index per merged spec.
WEBDRIVER_BIDI_CDDL = "{core_cddl_label}"

BIDI_EXTENSION_CDDL_FILES = [
{extension_cddl_labels}
]

BIDI_DFNS_FILES = [
{dfns_labels}
]

# Vendor grammars keyed by namespace (`moz` for `moz:` fields). Selenium copies, held until each
# can be pinned from its vendor's tree (Firefox: mozilla-central remote/doc/webdriver-bidi).
BIDI_VENDOR_CDDL_FILES = {{
{vendor_cddl_entries}
}}

def _webref_cddl_impl(_ctx):
    for name, filename, sha256 in _CDDL_FILES:
        http_file(
            name = name,
            downloaded_file_path = "spec.cddl",
            sha256 = sha256,
            url = _CDDL_BASE_URL + "/" + filename,
        )
    for name, filename, sha256 in _DFNS_FILES:
        http_file(
            name = name,
            downloaded_file_path = "dfns.json",
            sha256 = sha256,
            url = _DFNS_BASE_URL + "/" + filename,
        )
    http_file(
        name = "{bidi_repo_name}",
        downloaded_file_path = "index.html",
        sha256 = _BIDI_SPEC_HTML_SHA256,
        url = _BIDI_SPEC_HTML_URL,
    )
    return _ctx.extension_metadata(
        root_module_direct_deps = "all",
        root_module_direct_dev_deps = [],
        reproducible = True,
    )

webref_cddl_extension = module_extension(
    implementation = _webref_cddl_impl,
)
'''


def get(url, headers=None):
    r = http.request("GET", url, headers=headers)
    if r.status != 200:
        raise RuntimeError(f"Failed to download {url}: HTTP {r.status}")
    return r.data


def resolve_commit_for(repo, branch):
    data = get(f"https://api.github.com/repos/{repo}/commits/{branch}", API_HEADERS)
    return json.loads(data)["sha"]


def sha256_of(data):
    return hashlib.sha256(data).hexdigest()


def external_spec_urls(spec_html):
    """The specs the rendered core spec's "External specifications" section links to, without fragments."""
    section = re.search(r'id="external-specifications".*?<ol>(.*?)</ol>', spec_html, re.DOTALL)
    if not section:
        raise RuntimeError("Could not find the External specifications list in the rendered WebDriver BiDi spec")
    urls = [href.split("#")[0] for href in re.findall(r'href="(https?://[^"]+)"', section.group(1))]
    if not urls:
        raise RuntimeError("The External specifications list in the rendered WebDriver BiDi spec has no links")
    return urls


def spec_urls(entry):
    urls = {entry.get("url")}
    for key in ("nightly", "release"):
        if entry.get(key):
            urls.add(entry[key].get("url"))
    return {url.rstrip("/") for url in urls if url}


def resolve_specs(webref_commit, urls):
    """(shortname, cddl_filename) for the core spec and each linked spec, resolved through webref's index."""
    index = json.loads(get(WEBREF_RAW.format(commit=webref_commit, path="index.json")))["results"]
    by_shortname = {entry["shortname"]: entry for entry in index}
    entries = [by_shortname[CORE_SHORTNAME]]
    for url in urls:
        matches = [entry for entry in index if url.rstrip("/") in spec_urls(entry)]
        if len(matches) != 1:
            raise RuntimeError(f"Expected one webref spec for {url}, found {len(matches)}")
        entries.append(matches[0])

    specs = []
    for entry in entries:
        extracts = [extract["file"] for extract in entry.get("cddl", []) if extract["name"] == "all"]
        if len(extracts) != 1:
            raise RuntimeError(f"{entry['shortname']} has no single 'all' CDDL extract in webref: {entry.get('cddl')}")
        specs.append((entry["shortname"], Path(extracts[0]).name))
    return specs


def repo_names(shortname):
    stem = shortname.replace("-", "_")
    return f"{stem}_all_cddl", f"{stem}_dfns"


def pin_specs(webref_commit, specs):
    """(shortname, cddl_filename, cddl_sha256, dfns_sha256) per spec, hashed at the webref commit."""
    pinned = []
    for shortname, cddl_filename in specs:
        cddl_sha256 = sha256_of(get(WEBREF_RAW.format(commit=webref_commit, path=f"cddl/{cddl_filename}")))
        dfns_sha256 = sha256_of(get(WEBREF_RAW.format(commit=webref_commit, path=f"dfns/{shortname}.json")))
        pinned.append((shortname, cddl_filename, cddl_sha256, dfns_sha256))
    return pinned


def render(webref_commit, pinned, bidi_commit, bidi_sha256):
    cddl_entries, dfns_entries, extension_cddl_labels, dfns_labels = [], [], [], []
    core_cddl_label = None
    for shortname, cddl_filename, cddl_sha256, dfns_sha256 in pinned:
        cddl_repo, dfns_repo = repo_names(shortname)
        cddl_entries.append(f'    ("{cddl_repo}", "{cddl_filename}", "{cddl_sha256}"),')
        dfns_entries.append(f'    ("{dfns_repo}", "{shortname}.json", "{dfns_sha256}"),')
        cddl_label = f"@{cddl_repo}//file:spec.cddl"
        if shortname == CORE_SHORTNAME:
            core_cddl_label = cddl_label
        else:
            extension_cddl_labels.append(f'    "{cddl_label}",')
        dfns_labels.append(f'    "@{dfns_repo}//file:dfns.json",')
    return BZL_TEMPLATE.format(
        webref_commit=webref_commit,
        bidi_commit=bidi_commit,
        bidi_sha256=bidi_sha256,
        bidi_repo_name=BIDI_SPEC_REPO_NAME,
        cddl_entries="\n".join(cddl_entries),
        dfns_entries="\n".join(dfns_entries),
        core_cddl_label=core_cddl_label,
        extension_cddl_labels="\n".join(extension_cddl_labels),
        dfns_labels="\n".join(dfns_labels),
        vendor_cddl_entries=render_vendor_entries(),
    )


def render_vendor_entries():
    lines = []
    for namespace, labels in VENDOR_CDDL_FILES.items():
        lines.append(f'    "{namespace}": [')
        lines.extend(f'        "{label}",' for label in labels)
        lines.append("    ],")
    return "\n".join(lines)


def update_module(pinned):
    """Rewrite the extension's use_repo block in MODULE.bazel to name every pinned repo.

    Deliberately not `bazel mod tidy`: that also rewrites every other extension's block and
    reformats the file, which does not belong in a CDDL repin.
    """
    names = {BIDI_SPEC_REPO_NAME}
    for shortname, *_ in pinned:
        names.update(repo_names(shortname))
    repo_list = "\n".join(f'    "{name}",' for name in sorted(names))
    content = MODULE_FILE.read_text()
    content, count = re.subn(
        r"use_repo\(\n    webref_cddl_extension,\n.*?\n\)",
        f"use_repo(\n    webref_cddl_extension,\n{repo_list}\n)",
        content,
        flags=re.DOTALL,
    )
    if count != 1:
        raise RuntimeError(f"Expected exactly one webref_cddl_extension use_repo block, found {count}")
    MODULE_FILE.write_text(content)


def main():
    webref_commit = resolve_commit_for(WEBREF_REPO, WEBREF_BRANCH)
    bidi_commit = resolve_commit_for(BIDI_SPEC_REPO, BIDI_SPEC_BRANCH)
    print(f"Pinning {WEBREF_REPO}@{webref_commit} and {BIDI_SPEC_REPO}@{bidi_commit} ({BIDI_SPEC_FILE})")
    spec_html = get(BIDI_SPEC_RAW.format(commit=bidi_commit))
    specs = resolve_specs(webref_commit, external_spec_urls(spec_html.decode()))
    print(f"Merged specs: {', '.join(shortname for shortname, _ in specs)}")
    pinned = pin_specs(webref_commit, specs)

    BZL_FILE.write_text(render(webref_commit, pinned, bidi_commit, sha256_of(spec_html)))
    update_module(pinned)
    print(f"Updated {BZL_FILE.relative_to(root_dir)} and {MODULE_FILE.relative_to(root_dir)}")


if __name__ == "__main__":
    main()