build.wls

#!/usr/bin/env wolframscript
(* Self-hosting build: define MarkdownToNotebook from its own markdown source,
   then build and deploy *both* the forward converter (MarkdownToNotebook) and
   its inverse (NotebookToMarkdown) - each is a FunctionResource whose own
   markdown document inlines the shared MarkdownTools.wl plus its own .wl file.
   The only hand-written part is the tiny Definition-cell extractor below;
   everything else comes from the markdown. *)

(* the definition-notebook template version follows the kernel's
   DefinitionNotebookClient paclet; pull the latest so the generated notebook is
   not flagged "out of date" when opened in a newer front end. *)
Quiet @ PacletInstall["DefinitionNotebookClient", UpdateSites -> True]

(* The appearance is pinned per notebook, never on $FrontEnd (a front end setting can
   persist into the desktop app): the converter renders example thumbnails in its
   "LightDark" mode (light by default) and gives a resource notebook LightDark -> "Light". *)

dir = DirectoryName[$InputFileName]

(* phase 1: pull every `#| file: path` reference out of the "## Definition" section
   of the markdown document and Get each in order. The forward and inverse docs
   both inline MarkdownTools.wl first (shared protocol) and then their own .wl,
   so this loop is the entire bootstrap for either resource. *)
definitionSection[t_String] := First[
    StringCases[t, StartOfLine ~~ "## Definition" ~~ "\n" ~~ Shortest[s___] ~~ ("\n## " | EndOfString) :> s, 1],
    ""
]
definitionCells[t_String] := StringCases[
    definitionSection[t],
    "```" ~~ ("wl" | "wolfram" | "mathematica") ~~ "\n" ~~ Shortest[c___] ~~ "\n```" :> c
]
includeFiles[t_String] := FileNameJoin[{dir, StringTrim[#]}] & /@
    StringCases[definitionSection[t], StartOfLine ~~ "#|" ~~ Whitespace... ~~ "file:" ~~ p : Except["\n"].. :> p]

loadDefinition[src_String] := Block[{text = Import[src, "Text"], includes},
    includes = includeFiles[text];
    If[ includes =!= {},
        Print["  including: ", FileNameTake /@ includes];
        Scan[Get, includes]
        ,
        Block[{boot = CreateFile[]},
            Export[boot, StringRiffle[definitionCells[text], "\n\n"], "Text"];
            Get[boot]
        ]
    ]
]

(* phase 2-5: convert one resource document end to end: define it from its own
   .wl includes, run the forward converter on it (cold + warm to demonstrate
   the persistent example cache), write the .nb, deploy it publicly to the
   cloud at the chosen stable URL. *)
buildOne[mdName_String, entrySymbol_Symbol, cloudPath_String] := Block[
    {src, text, coldTime, result, nb, warmTime, outNb, deployed, failedOut},
    src = FileNameJoin[{dir, mdName}];
    text = Import[src, "Text"];
    Print[""];
    Print["build: ", mdName];
    loadDefinition[src];
    If[ DownValues[entrySymbol] === {},
        Print["  warn: ", SymbolName[entrySymbol], " has no definition after include"]
    ];
    Print["  converting (cold run)"];
    {coldTime, result} = AbsoluteTiming @ MarkdownToNotebook[src, "Association"];
    nb = result["Notebook"];
    Print["    Template:    ", result["Template"]];
    Print["    Name:        ", result["Metadata"]["Name"]];
    Print["    Sections:    ", result["Sections"]];
    Print["    Output cells:   ", Count[nb, Cell[_, "Output", ___], Infinity], " (evaluated examples)"];
    Print["    Unfilled slots: ", Count[nb, _TemplateSlot, Infinity]];
    Print["    cold run:    ", coldTime, " s"];
    warmTime = First @ AbsoluteTiming @ MarkdownToNotebook[src, "Association"];
    Print["    warm run:    ", warmTime, " s (", Length @ PersistentObjects["MarkdownToNotebook/ExampleOutput/*", "Local"], " cached)"];
    outNb = MarkdownToNotebook[src, FileNameJoin[{dir, FileBaseName[src] <> ".nb"}]];
    Print["    wrote:       ", FileNameTake[outNb], " (", Head @ Quiet @ Get[outNb], ")"];
    (* A network flake mid-build silently ships a broken example: the Applications
       cells convert documents by URL, and a failed fetch lands $Failed in the notebook
       as that cell's output, where it stays until a reviewer finds it. A clean build has
       none - the one deliberate failure case (a mistyped path) parses as raw markdown
       and returns a notebook - so treat any as a reason not to publish this build. *)
    failedOut = Count[nb, Cell[BoxData["$Failed"], "Output", ___], Infinity];
    Print["    $Failed outputs: ", failedOut];
    If[ TrueQ[$CloudConnected] && failedOut === 0,
        deployed = UsingFrontEnd @ Block[{nbo, ro, rf, defBytes},
            nbo = NotebookOpen[File[outNb]];
            CurrentValue[nbo, CreateCellID] = True;
            SelectionMove[nbo, All, Notebook];
            FrontEndTokenExecute[nbo, "Save"];
            (* Quiet swallows ExpandFileName::badfile that the framework's
               fastPathHash cache fires when an Example notebook reference
               resolves to None during scraping. The message looks fatal
               but isn't; with it surfaced, the whole ResourceObject call
               returns $Failed and the deploy aborts. *)
            ro = Quiet @ ResourceObject[nbo];
            rf = ResourceFunction[ro];
            NotebookClose[nbo];
            (* A resource whose definitions have swallowed a dependency's private
               state is published fine and then cannot be used: the fetching kernel
               downloads the blob and ResourceFunction[url] never returns. Both
               converters serialize to tens of KB, so anything past this bound means
               a bare context token in the definitions anchored a paclet (see
               parserLoadedQ here, the n2m* wrappers in the inverse) and dragged its
               state in - the warm compiled parser cache alone is ~430 MB. Refuse to
               publish rather than replace a working resource with a hanging one. *)
            defBytes = Length @ ResourceFunction[rf, "DefinitionData"];
            If[ defBytes > 1000000,
                Print["    definition:  ", defBytes, " bytes"];
                Print["    ABORT: definitions carry a dependency's state; refusing",
                      " to publish an unloadable resource."];
                $Failed
                ,
                Print["    definition:  ", defBytes, " bytes"];
                CloudDeploy[rf, CloudObject[cloudPath], Permissions -> "Public"]
            ]
        ];
        Print["    deployed:    ", Replace[deployed, {
            co_CloudObject :> First[co],
            ro_ResourceObject :> ro["DocumentationLink"],
            a_Association :> Lookup[a, "ResultLink", a]
        }]]
        ,
        Print["    skipped:     ", If[! TrueQ[$CloudConnected],
            "not cloud-connected",
            ToString[failedOut] <> " example(s) evaluated to $Failed - refusing to publish a build with a broken example"]]
    ]
]

(* clear the per-cell evaluation cache once up front so the cold-run numbers
   are honest; the warm-run inside buildOne measures the hit rate against the
   entries this cold run just wrote. *)
Print["build: clearing example cache"]
Scan[DeleteObject, PersistentObjects["MarkdownToNotebook/ExampleOutput/*", "Local"]]

Print["build: connecting cloud"]
Needs["DefinitionNotebookClient`"]
If[ Not[TrueQ[$CloudConnected]], Quiet @ CloudConnect[] ]
If[ Not[TrueQ[$CloudConnected]], Print["  warn: not cloud-connected, deploy steps will be skipped"] ]

buildOne["MarkdownToNotebook.md", MarkdownToNotebook, "DeployedResources/Function/MarkdownToNotebook"]
buildOne["NotebookToMarkdown.md", NotebookToMarkdown, "DeployedResources/Function/NotebookToMarkdown"]

Print[""]
Print["build: done (source: github.com/WolframInstitute/MarkdownToNotebook)"]