From d4b1e80f4439d9f400aeb6bf1cd2e7870faaeaa1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 18:26:41 +0000 Subject: [PATCH 01/24] Add a JSPaint mouse-only environment and a circle verifier JSPaintEnv drives an unmodified JSPaint checkout in headless Chromium through Playwright. The observation is a canvas screenshot with the cursor drawn on; the actions are eight fixed-length moves plus pen down, pen up and done, sent as real pointer events. The same moves are exposed as a mouse-only tool API (TOOLS / call_tool) for tool-calling agents. score_circle fits a least-squares circle to the canvas ink and scores fit, angular coverage, cleanliness, closure and size. Rings and the octagon the eight moves can draw pass; lines, arcs, squares, ellipses, discs, dots and scribbles fail. examples/jspaint_circle.py plays the model, a scripted circle expert and a random policy on the same seeds and writes trajectories, GIFs and a summary. Zero-shot thaitea/laya-vision scores 0 (near-uniform actions, early DONE); the expert scores 0.76 with a 100% pass rate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- examples/jspaint_circle.py | 101 +++++++++ laya/circle_verifier.py | 102 +++++++++ laya/games.py | 24 +++ laya/paintenv.py | 362 ++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + pyproject.toml | 3 + site-docs/how-to/jspaint-env.md | 85 ++++++++ tests/test_circle_verifier.py | 74 +++++++ tests/test_paintenv.py | 92 ++++++++ 9 files changed, 844 insertions(+) create mode 100644 examples/jspaint_circle.py create mode 100644 laya/circle_verifier.py create mode 100644 laya/paintenv.py create mode 100644 site-docs/how-to/jspaint-env.md create mode 100644 tests/test_circle_verifier.py create mode 100644 tests/test_paintenv.py diff --git a/examples/jspaint_circle.py b/examples/jspaint_circle.py new file mode 100644 index 00000000..7a043047 --- /dev/null +++ b/examples/jspaint_circle.py @@ -0,0 +1,101 @@ +"""Ask Laya Vision to draw a circle in JSPaint with the mouse only, and score it. + +Each step a screenshot of the JSPaint canvas (with the cursor marked on it) goes into the model as the image, and a +`choice` question over mouse actions (eight moves, pen down, pen up, done) picks the next move, which is sent to the +real app as pointer events. At the end the circle verifier scores the true canvas pixels. The scripted expert and +a random policy play the same seeds as reference points, and the model's score is also reported normalized, +(model - random) / (expert - random), as in the games suite. + + pip install -e . playwright pillow + git clone https://github.com/r33drichards/jspaint ../jspaint + python examples/jspaint_circle.py --jspaint ../jspaint --policies expert,random # no model needed + python examples/jspaint_circle.py --jspaint ../jspaint --episodes 5 # + thaitea/laya-vision + python examples/jspaint_circle.py --model checkpoints/my-run --revision --device cuda + +Chromium: set LAYA_CHROMIUM (or --chromium) to an existing binary instead of running `playwright install`. +Each run writes a new directory under --out (create-only): summary.json, and per policy and episode a +trajectory.jsonl (action, cursor, pen, and the model's action probabilities), final.png and episode.gif. +""" +import argparse +import json +import os +import time + +from laya.paintenv import JSPaintEnv, JSPaintServer, circle_expert, model_policy, play_episodes, random_policy + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--jspaint", default=os.path.join(os.path.dirname(__file__), "..", "..", "jspaint"), + help="path to a JSPaint checkout (default: a sibling of this repo)") + ap.add_argument("--policies", default="model,expert,random", help="comma list of model, expert, random") + ap.add_argument("--episodes", type=int, default=5) + ap.add_argument("--seed", type=int, default=900_000) + ap.add_argument("--max-steps", type=int, default=80) + ap.add_argument("--step-px", type=int, default=24) + ap.add_argument("--model", default="thaitea/laya-vision") + ap.add_argument("--revision", default=None, help="pin the Hub revision of --model") + ap.add_argument("--device", default=None) + ap.add_argument("--chromium", default=None, help="Chromium executable (default: LAYA_CHROMIUM or Playwright's)") + ap.add_argument("--headed", action="store_true", help="show the browser window") + ap.add_argument("--out", default="results/jspaint") + args = ap.parse_args() + + out_dir = os.path.join(args.out, time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(out_dir, exist_ok=False) + names = [p.strip() for p in args.policies.split(",") if p.strip()] + summary = {"task": "circle", "jspaint": os.path.abspath(args.jspaint), "args": vars(args), "policies": {}} + + with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, max_steps=args.max_steps, + step_px=args.step_px, headless=not args.headed, + executable_path=args.chromium, keep_frames=True) as env: + for name in names: + if name == "model": + from laya import load_vlm + + policy = model_policy(load_vlm(args.model, revision=args.revision, device=args.device)) + elif name == "expert": + policy = circle_expert() + elif name == "random": + policy = random_policy(args.seed) + else: + raise SystemExit("unknown policy %r" % name) + + rows = [] + + def on_step(env, action): + row = dict(env.trajectory[-1], episode=env.seed) + if name == "model": + row["probabilities"] = policy.last["probabilities"] + rows.append(row) + if env.done: + ep_dir = os.path.join(out_dir, name, "seed%d" % env.seed) + os.makedirs(ep_dir) + with open(os.path.join(ep_dir, "trajectory.jsonl"), "w") as f: + f.writelines(json.dumps(r) + "\n" for r in rows) + env.frames[-1].save(os.path.join(ep_dir, "final.png")) + env.frames[0].save(os.path.join(ep_dir, "episode.gif"), save_all=True, + append_images=env.frames[1:], duration=120, loop=0) + rows.clear() + + t = time.time() + res = play_episodes(env, policy, args.episodes, seed=args.seed, on_step=on_step) + res["seconds"] = round(time.time() - t, 1) + if name == "model": + res["model"], res["provenance"] = args.model, policy.provenance + summary["policies"][name] = res + print("%-7s mean score %.3f pass rate %.2f mean steps %.1f (%.0fs)" % ( + name, res["mean_score"], res["pass_rate"], res["mean_steps"], res["seconds"])) + + pol = summary["policies"] + if {"model", "expert", "random"} <= set(pol): + e, r = pol["expert"]["mean_score"], pol["random"]["mean_score"] + summary["normalized_model_score"] = (pol["model"]["mean_score"] - r) / (e - r) if e > r else None + print("model normalized score (model - random) / (expert - random): %s" % summary["normalized_model_score"]) + with open(os.path.join(out_dir, "summary.json"), "w") as f: + json.dump(summary, f, indent=2, default=str) + print("wrote %s" % out_dir) + + +if __name__ == "__main__": + main() diff --git a/laya/circle_verifier.py b/laya/circle_verifier.py new file mode 100644 index 00000000..b7a4a993 --- /dev/null +++ b/laya/circle_verifier.py @@ -0,0 +1,102 @@ +"""Score how well a canvas shows one drawn circle: the verifier behind the JSPaint "draw a circle" task. + +Pure numpy, so it scores any RGB array (the JSPaint canvas, a PIL image, a synthetic test image) without a browser. +The steps: + +1. **Ink**: pixels that differ from the background colour by more than ``ink_threshold`` in any channel. +2. **Circle fit**: an algebraic least-squares (Kasa) fit of ``x^2 + y^2 + a x + b y + c = 0`` to the ink pixels, giving + the centre ``(cx, cy)`` and radius ``r``. +3. **Components**, each in [0, 1]: + + * ``fit``: how tightly the ink hugs the fitted circle, ``1 - rms(|d - r|) / (fit_tol * r)``, where ``d`` is each + ink pixel's distance from the centre; + * ``coverage``: the fraction of ``bins`` equal angular sectors around the centre that hold ink on the ring, so an + arc or a C-shape loses the missing sectors; + * ``clean``: the fraction of ink on the ring band (within ``band * r`` of it), so scribbles, fills and stray + strokes count against the drawing; + * ``closure``: 1 when no angular gap is wider than ``max_gap_deg``, falling linearly to 0 at ``max_gap_deg + 90``, + so an unclosed arc scores low even when what was drawn is round; + * ``size``: 1 when ``r`` is between ``min_radius`` and ``max_radius`` of the shorter canvas side and the centre is + on the canvas, else 0 (a dot or a straight line fits a tiny or enormous "circle"). + +4. ``score = size * fit * coverage * clean * closure``. ``passed`` needs ``score >= pass_score`` and no angular gap + wider than ``max_gap_deg`` (the circle is closed). + +The tolerances are loose enough that a circle walked with only eight fixed-length moves (a rough polygon, which is +what the environment's action set can draw) still passes, and tight enough that lines, arcs, squares, filled discs, +dots and scribbles fail; ``tests/test_circle_verifier.py`` pins both sides. +""" +from typing import Dict, Sequence + +import numpy as np + + +def ink_mask(pixels, bg: Sequence[int] = (255, 255, 255), ink_threshold: int = 60) -> np.ndarray: + """A boolean ``(h, w)`` mask of pixels whose colour differs from ``bg`` by more than ``ink_threshold``.""" + arr = np.asarray(pixels) + if arr.ndim == 2: + arr = np.repeat(arr[..., None], 3, axis=2) + arr = arr[..., :3].astype(np.int16) + return (np.abs(arr - np.asarray(bg, dtype=np.int16)).max(axis=2) > ink_threshold) + + +def fit_circle(xs: np.ndarray, ys: np.ndarray): + """Least-squares (Kasa) circle through the points: ``(cx, cy, r)``.""" + xs, ys = xs.astype(np.float64), ys.astype(np.float64) + a = np.stack([xs, ys, np.ones_like(xs)], axis=1) + b = xs ** 2 + ys ** 2 + (p, q, c), *_ = np.linalg.lstsq(a, b, rcond=None) + cx, cy = p / 2.0, q / 2.0 + return float(cx), float(cy), float(np.sqrt(max(c + cx ** 2 + cy ** 2, 0.0))) + + +def _max_gap_deg(occupied: np.ndarray) -> float: + """Widest run of empty angular bins (wrapping around), in degrees.""" + n = len(occupied) + if not occupied.any(): + return 360.0 + run = best = 0 + for v in np.concatenate([occupied, occupied]): + run = 0 if v else run + 1 + best = max(best, run) + return min(best, n) * 360.0 / n + + +def score_circle(pixels, bg: Sequence[int] = (255, 255, 255), ink_threshold: int = 60, min_ink: int = 30, + fit_tol: float = 0.2, band: float = 0.2, bins: int = 36, min_radius: float = 0.08, + max_radius: float = 0.5, max_gap_deg: float = 30.0, pass_score: float = 0.6) -> Dict: + """Score an ``(h, w, 3)`` canvas for one drawn circle. Returns ``score`` in [0, 1], ``passed``, every component, + and the fitted ``cx, cy, r`` (see the module docstring).""" + mask = ink_mask(pixels, bg, ink_threshold) + h, w = mask.shape + ys, xs = np.nonzero(mask) + out = {"score": 0.0, "passed": False, "ink": int(len(xs)), "fit": 0.0, "coverage": 0.0, "clean": 0.0, + "closure": 0.0, "size": 0.0, "max_gap_deg": 360.0, "cx": None, "cy": None, "r": None} + if len(xs) < min_ink: + return out + cx, cy, r = fit_circle(xs, ys) + out.update(cx=round(cx, 2), cy=round(cy, 2), r=round(r, 2)) + side = min(h, w) + size_ok = min_radius * side <= r <= max_radius * side and 0 <= cx < w and 0 <= cy < h + if r <= 0: + return out + d = np.hypot(xs - cx, ys - cy) + dev = np.abs(d - r) + fit = float(np.clip(1.0 - np.sqrt(np.mean(dev ** 2)) / (fit_tol * r), 0.0, 1.0)) + on_ring = dev <= max(band * r, 3.0) + clean = float(on_ring.mean()) + ang = np.arctan2(ys[on_ring] - cy, xs[on_ring] - cx) + idx = ((ang + np.pi) / (2 * np.pi) * bins).astype(int) % bins + occupied = np.zeros(bins, dtype=bool) + occupied[idx] = True + coverage = float(occupied.mean()) + gap = _max_gap_deg(occupied) + closure = float(np.clip(1.0 - (gap - max_gap_deg) / 90.0, 0.0, 1.0)) + score = float(size_ok) * fit * coverage * clean * closure + out.update(score=round(score, 4), fit=round(fit, 4), coverage=round(coverage, 4), clean=round(clean, 4), + closure=round(closure, 4), size=float(size_ok), max_gap_deg=round(gap, 1), + passed=bool(score >= pass_score and gap <= max_gap_deg)) + return out + + +__all__ = ["ink_mask", "fit_circle", "score_circle"] diff --git a/laya/games.py b/laya/games.py index f3a0b65a..2592adca 100644 --- a/laya/games.py +++ b/laya/games.py @@ -77,6 +77,30 @@ def snake_question() -> Dict: }} +PAINT_ACTIONS = { + "UP": "move the cursor up", "DOWN": "move the cursor down", "LEFT": "move the cursor left", + "RIGHT": "move the cursor right", "UP_LEFT": "move the cursor diagonally up and to the left", + "UP_RIGHT": "move the cursor diagonally up and to the right", + "DOWN_LEFT": "move the cursor diagonally down and to the left", + "DOWN_RIGHT": "move the cursor diagonally down and to the right", + "PEN_DOWN": "press the mouse button to start drawing", "PEN_UP": "release the mouse button to stop drawing", + "DONE": "the drawing is finished", +} +PAINT_GOALS = {"circle": "draw one round, closed circle, about as big as a third of the canvas height"} + + +def paint_question(task: str = "circle") -> Dict: + """The question for ``laya.paintenv.JSPaintEnv``: a paint canvas, a red cursor, and mouse-only actions.""" + return {"action": { + "type": "choice", + "instructions": "You are using a paint program with only the mouse. The white area is the canvas and the " + "red mark is the mouse cursor: a hollow ring with a cross means the button is up, a filled " + "dot means it is held down and moving draws a black line. Your task: %s. Which mouse action " + "should you take now?" % PAINT_GOALS[task], + "criteria": dict(PAINT_ACTIONS), + }} + + CONTROL_GOALS = { "CartPole": "A pole is hinged on a cart; push the cart left or right to keep the pole upright and the cart " "on screen.", diff --git a/laya/paintenv.py b/laya/paintenv.py new file mode 100644 index 00000000..37457798 --- /dev/null +++ b/laya/paintenv.py @@ -0,0 +1,362 @@ +"""JSPaint as an RL environment: the model sees screenshots of the real app and drives it with the mouse only. + +`JSPaint `_ (an MS Paint clone) runs unmodified in headless Chromium through +Playwright; ``JSPaintServer`` serves a local checkout as static files. Each episode: + +* **reset** clears the canvas to white (``api_for_cypress_tests.reset_for_next_test()``), selects the Brush tool + (so strokes are 4 px wide and visible) and puts the cursor at a seeded point on the canvas; +* **observation** is a screenshot of the canvas with the cursor drawn on it (headless screenshots have no OS cursor): + a red ring when the pen is up, a filled red dot when it is down. ``note()`` carries the pen state and step count + as text; +* **actions** are mouse-only. ``ACTIONS`` is the discrete set the image model chooses from (eight ``step_px`` moves, + ``PEN_DOWN``, ``PEN_UP``, ``DONE``); each maps onto the tool API ``move_mouse(dx, dy)`` / ``mouse_down()`` / + ``mouse_up()``, which sends real pointer events to the page. ``TOOLS`` describes that API as JSON-schema tools, so + a tool-calling agent can drive the same environment with ``call_tool``; +* **reward** is the task verifier on the true canvas pixels (``canvas_pixels()``, read from the page, so the cursor + overlay never counts as ink): ``laya.circle_verifier.score_circle`` for ``task="circle"``. It is paid at the end of + the episode, or as the per-step change in score with ``reward="shaped"``. An episode ends on ``DONE`` or after + ``max_steps``. + +``circle_expert`` (a scripted policy that walks a circle with the same eight moves), ``random_policy`` and +``model_policy`` (``VLMAgent.predict`` on the screenshot) are the reference points; ``play_episodes`` runs any of +them. ``examples/jspaint_circle.py`` is the command-line runner. + +Needs ``pip install playwright pillow`` and a Chromium: set ``executable_path`` (or ``LAYA_CHROMIUM``) to use an +existing binary instead of ``playwright install``. +""" +import base64 +import functools +import http.server +import io +import math +import os +import random +import threading +from collections import Counter +from typing import Dict, List, Optional, Tuple + +import numpy as np + +MOVES = { + "UP": (0, -1), "DOWN": (0, 1), "LEFT": (-1, 0), "RIGHT": (1, 0), + "UP_LEFT": (-1, -1), "UP_RIGHT": (1, -1), "DOWN_LEFT": (-1, 1), "DOWN_RIGHT": (1, 1), +} +ACTIONS = tuple(MOVES) + ("PEN_DOWN", "PEN_UP", "DONE") +TASKS = {"circle": "Draw a circle on the canvas."} + +TOOLS = [ + {"name": "move_mouse", "description": "Move the mouse by (dx, dy) canvas pixels. If the button is held down, " + "this draws a brush stroke along the way.", + "input_schema": {"type": "object", "properties": {"dx": {"type": "integer"}, "dy": {"type": "integer"}}, + "required": ["dx", "dy"]}}, + {"name": "mouse_down", "description": "Press the left mouse button (start drawing).", + "input_schema": {"type": "object", "properties": {}}}, + {"name": "mouse_up", "description": "Release the left mouse button (stop drawing).", + "input_schema": {"type": "object", "properties": {}}}, + {"name": "screenshot", "description": "Return a screenshot of the canvas with the cursor marked.", + "input_schema": {"type": "object", "properties": {}}}, +] + +CURSOR = (230, 30, 30) + + +def default_chromium() -> Optional[str]: + """``LAYA_CHROMIUM``, else the Claude Code sandbox's preinstalled Chromium, else ``None`` (Playwright's own).""" + path = os.environ.get("LAYA_CHROMIUM") or "/opt/pw-browsers/chromium" + return path if os.path.exists(path) else None + + +class JSPaintServer: + """Serve a JSPaint checkout on ``127.0.0.1`` (a free port) from a background thread; ``.url`` is the app.""" + + def __init__(self, root: str): + if not os.path.exists(os.path.join(root, "index.html")): + raise FileNotFoundError("no JSPaint checkout at %r (index.html missing)" % root) + + class Quiet(http.server.SimpleHTTPRequestHandler): + def log_message(self, *args): + pass + + self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), functools.partial(Quiet, directory=root)) + self.url = "http://127.0.0.1:%d/" % self.httpd.server_port + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + + def close(self) -> None: + self.httpd.shutdown() + self.httpd.server_close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + +class JSPaintEnv: + """One headless JSPaint page; ``reset`` starts an episode and ``step(action)`` plays one of ``ACTIONS``.""" + + def __init__(self, url: str, task: str = "circle", step_px: int = 24, max_steps: int = 80, + reward: str = "terminal", headless: bool = True, executable_path: Optional[str] = None, + viewport: Tuple[int, int] = (800, 600), keep_frames: bool = False): + if task not in TASKS: + raise ValueError("unknown task %r (one of %s)" % (task, ", ".join(TASKS))) + if reward not in ("terminal", "shaped"): + raise ValueError("reward must be 'terminal' or 'shaped'") + from playwright.sync_api import sync_playwright + + self.url, self.task, self.step_px, self.max_steps, self.reward_mode = url, task, step_px, max_steps, reward + self.keep_frames = keep_frames + self._pw = sync_playwright().start() + self.browser = self._pw.chromium.launch(headless=headless, + executable_path=executable_path or default_chromium()) + self.page = self.browser.new_page(viewport={"width": viewport[0], "height": viewport[1]}) + # Only the app itself: no fonts, analytics or update checks from the network. + self.page.route("**/*", lambda route: route.continue_() if route.request.url.startswith(url) + else route.abort()) + self.page.goto(url) + self.page.wait_for_function("window.api_for_cypress_tests") + self.episode, self.steps, self.pen, self.finished = -1, 0, False, False + self.cursor = (0.0, 0.0) + self.trajectory: List[Dict] = [] + self.frames: List = [] + self._last_score = 0.0 + self.result: Optional[Dict] = None + + # -- episode ---------------------------------------------------------------------------------------------- + def reset(self, seed: int = 0): + """Blank canvas, Brush tool, cursor at a seeded point in the middle half of the canvas. Returns the + observation.""" + self.page.mouse.up() + self.page.evaluate("api_for_cypress_tests.reset_for_next_test()") + self.page.click(".tool[title='Brush']") + r = self.page.evaluate("(() => { const r = document.querySelector('.main-canvas').getBoundingClientRect();" + " return [r.x, r.y, r.width, r.height]; })()") + self.origin, self.width, self.height = (r[0], r[1]), int(r[2]), int(r[3]) + rng = random.Random(seed) + self.cursor = (rng.uniform(0.25, 0.75) * self.width, rng.uniform(0.25, 0.75) * self.height) + self.page.mouse.move(*self._client(self.cursor)) + self.episode += 1 + self.seed, self.steps, self.pen, self.finished = seed, 0, False, False + self.trajectory, self.frames, self._last_score, self.result = [], [], 0.0, None + obs = self.render() + if self.keep_frames: + self.frames.append(obs) + return obs + + @property + def done(self) -> bool: + return self.finished or self.steps >= self.max_steps + + def step(self, action: str): + """Play one of ``ACTIONS``. Returns ``(observation, reward, done, info)``; ``info["verifier"]`` holds the full + verifier result once the episode is over.""" + if action not in ACTIONS: + raise ValueError("unknown action %r" % action) + if self.done: + raise RuntimeError("episode is over; call reset()") + if action in MOVES: + dx, dy = MOVES[action] + self.move_mouse(dx * self.step_px, dy * self.step_px) + elif action == "PEN_DOWN": + self.mouse_down() + elif action == "PEN_UP": + self.mouse_up() + else: + self.finished = True + self.steps += 1 + self.trajectory.append({"step": self.steps, "action": action, "x": round(self.cursor[0], 1), + "y": round(self.cursor[1], 1), "pen": self.pen}) + info: Dict = {} + reward = 0.0 + if self.done: + self.mouse_up() + self.result = self.verify() + info["verifier"] = self.result + reward = self.result["score"] - (self._last_score if self.reward_mode == "shaped" else 0.0) + elif self.reward_mode == "shaped": + score = self.verify()["score"] + reward, self._last_score = score - self._last_score, score + obs = self.render() + if self.keep_frames: + self.frames.append(obs) + return obs, reward, self.done, info + + def verify(self) -> Dict: + from laya.circle_verifier import score_circle + + return score_circle(self.canvas_pixels()) + + # -- mouse-only tool API ------------------------------------------------------------------------------------ + def _client(self, xy) -> Tuple[float, float]: + return self.origin[0] + xy[0], self.origin[1] + xy[1] + + def move_mouse(self, dx: float, dy: float) -> None: + """Move by ``(dx, dy)`` canvas pixels, clamped to the canvas; draws while the button is down.""" + x = min(max(self.cursor[0] + dx, 1.0), self.width - 2.0) + y = min(max(self.cursor[1] + dy, 1.0), self.height - 2.0) + self.cursor = (x, y) + self.page.mouse.move(*self._client(self.cursor), steps=4 if self.pen else 1) + + def mouse_down(self) -> None: + if not self.pen: + self.page.mouse.down() + self.pen = True + + def mouse_up(self) -> None: + if self.pen: + self.page.mouse.up() + self.pen = False + + def call_tool(self, name: str, args: Optional[Dict] = None): + """Run one of ``TOOLS`` by name (for tool-calling agents); ``screenshot`` returns the observation.""" + args = args or {} + if name == "move_mouse": + self.move_mouse(float(args["dx"]), float(args["dy"])) + elif name == "mouse_down": + self.mouse_down() + elif name == "mouse_up": + self.mouse_up() + elif name == "screenshot": + return self.render() + else: + raise ValueError("unknown tool %r" % name) + return None + + # -- observations -------------------------------------------------------------------------------------------- + def screenshot(self, full_page: bool = False): + """A PIL screenshot of the canvas (or the whole app with ``full_page``), without the cursor.""" + from PIL import Image + + clip = None if full_page else {"x": self.origin[0], "y": self.origin[1], "width": self.width, + "height": self.height} + return Image.open(io.BytesIO(self.page.screenshot(clip=clip))).convert("RGB") + + def render(self, full_page: bool = False): + """The observation: a screenshot with the cursor drawn on it (ring = pen up, dot = pen down).""" + from PIL import ImageDraw + + img = self.screenshot(full_page) + x, y = self._client(self.cursor) if full_page else self.cursor + d = ImageDraw.Draw(img) + if self.pen: + d.ellipse([x - 5, y - 5, x + 5, y + 5], fill=CURSOR) + else: + d.ellipse([x - 7, y - 7, x + 7, y + 7], outline=CURSOR, width=2) + d.line([x - 11, y, x + 11, y], fill=CURSOR) + d.line([x, y - 11, x, y + 11], fill=CURSOR) + return img + + def note(self) -> str: + return "Task: %s The pen is %s. Step %d of %d." % (TASKS[self.task], "down (drawing)" if self.pen else + "up (not drawing)", self.steps + 1, self.max_steps) + + def canvas_pixels(self) -> np.ndarray: + """The true canvas as an ``(h, w, 3)`` uint8 array, read from the page (transparent pixels read as white).""" + from PIL import Image + + data = self.page.evaluate("main_canvas.toDataURL('image/png')") + img = Image.open(io.BytesIO(base64.b64decode(data.split(",", 1)[1]))).convert("RGBA") + white = Image.new("RGBA", img.size, (255, 255, 255, 255)) + return np.asarray(Image.alpha_composite(white, img).convert("RGB")) + + def close(self) -> None: + self.browser.close() + self._pw.stop() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + +# -- policies ------------------------------------------------------------------------------------------------------ +def _toward(pos, target) -> str: + """The move whose unit direction best matches ``target - pos``.""" + vx, vy = target[0] - pos[0], target[1] - pos[1] + return max(MOVES, key=lambda a: (MOVES[a][0] * vx + MOVES[a][1] * vy) / math.hypot(*MOVES[a])) + + +def circle_expert(radius_frac: float = 0.3): + """A scripted policy: walk (pen up) to the rightmost point of a circle centred on the canvas, press, trace one + full turn plus a little overlap with the eight moves (each step heads for the point one step further round the + circle), release, ``DONE``. Deterministic given the environment state.""" + state: Dict = {} + + def policy(env: JSPaintEnv) -> str: + if state.get("episode") != (id(env), env.episode): + cx, cy = env.width / 2.0, env.height / 2.0 + r = radius_frac * min(env.width, env.height) + state.clear() + state.update(episode=(id(env), env.episode), c=(cx, cy), r=r, phase="approach", turned=0.0, prev=None) + cx, cy = state["c"] + r, pos, step = state["r"], env.cursor, env.step_px + if state["phase"] == "approach": + start = (cx + r, cy) + if math.dist(pos, start) > step * 0.75: + return _toward(pos, start) + state["phase"] = "trace" + state["prev"] = math.atan2(pos[1] - cy, pos[0] - cx) + return "PEN_DOWN" + if state["phase"] == "trace": + ang = math.atan2(pos[1] - cy, pos[0] - cx) + delta = (ang - state["prev"] + math.pi) % (2 * math.pi) - math.pi + state["turned"] += delta + state["prev"] = ang + if state["turned"] < 2 * math.pi + 0.3: + nxt = ang + step / r + return _toward(pos, (cx + r * math.cos(nxt), cy + r * math.sin(nxt))) + state["phase"] = "lift" + return "PEN_UP" + return "DONE" + + return policy + + +def random_policy(seed: int = 0): + rng = random.Random(seed) + return lambda env: rng.choice(ACTIONS) + + +class ModelPolicy: + """The model's most likely action from the screenshot plus the text note, via ``predict``. ``last`` keeps the + latest answer (probabilities over ``ACTIONS``) for logging.""" + + def __init__(self, agent, task: str = "circle"): + from laya.games import paint_question + + self.agent, self.question, self.last, self.provenance = agent, paint_question(task), None, None + + def __call__(self, env: JSPaintEnv) -> str: + out = self.agent.predict({"image": env.render(), "note": env.note()}, self.question) + self.last, self.provenance = out["answers"]["action"], out.get("provenance") + return self.last["choice"] + + +def model_policy(agent, task: str = "circle") -> ModelPolicy: + return ModelPolicy(agent, task) + + +def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step=None) -> Dict: + """Play ``episodes`` seeded episodes (seed ``seed + i``) with ``policy(env) -> action``. ``on_step(env, action)`` + is called after each step (for logging). Returns per-episode verifier results and the summary.""" + counts, eps = Counter(), [] + for i in range(episodes): + env.reset(seed + i) + while not env.done: + a = policy(env) + counts[a] += 1 + env.step(a) + if on_step: + on_step(env, a) + eps.append({"seed": seed + i, "steps": env.steps, "ended": "done" if env.finished else "capped", + **env.result}) + scores = [e["score"] for e in eps] + return {"task": env.task, "episodes": episodes, "seed": seed, "step_px": env.step_px, "max_steps": env.max_steps, + "mean_score": float(np.mean(scores)), "median_score": float(np.median(scores)), + "pass_rate": float(np.mean([e["passed"] for e in eps])), + "mean_steps": float(np.mean([e["steps"] for e in eps])), "actions": dict(counts), "results": eps} + + +__all__ = ["ACTIONS", "MOVES", "TASKS", "TOOLS", "JSPaintServer", "JSPaintEnv", "circle_expert", "random_policy", + "ModelPolicy", "model_policy", "play_episodes", "default_chromium"] diff --git a/mkdocs.yml b/mkdocs.yml index 5440cdc2..ab8aeba9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - Run jobs on Modal: how-to/run-on-modal.md - Evaluate a checkpoint: how-to/evaluate.md - Play games with a checkpoint: how-to/play-games.md + - Draw in JSPaint with the mouse: how-to/jspaint-env.md - Concepts: - Overview: concepts/overview.md diff --git a/pyproject.toml b/pyproject.toml index 9a257ca2..6557e710 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,9 @@ dependencies = [ "numpy>=1.20.0", ] +[project.optional-dependencies] +paint = ["playwright", "pillow"] + [project.urls] Homepage = "https://github.com/r33drichards/laya-vision" Model = "https://huggingface.co/thaitea/laya-vision-smolvlm-256m" diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md new file mode 100644 index 00000000..5f8a53e7 --- /dev/null +++ b/site-docs/how-to/jspaint-env.md @@ -0,0 +1,85 @@ +# Draw in JSPaint with the mouse + +This environment gives the model a real paint program, [JSPaint](https://github.com/r33drichards/jspaint), and a +task such as "draw a circle". The model can only use the mouse. Each step it sees a screenshot of the canvas and +picks one mouse action, which is sent to the app as real pointer events. When the episode ends, a verifier scores +the pixels on the canvas. + +The environment is in `laya/paintenv.py`, the verifier in `laya/circle_verifier.py`, and the runner in +[`examples/jspaint_circle.py`](https://github.com/r33drichards/laya-vision/blob/main/examples/jspaint_circle.py). + +## Run it + +```bash +pip install -e ".[paint]" +git clone https://github.com/r33drichards/jspaint ../jspaint +python examples/jspaint_circle.py --jspaint ../jspaint --policies expert,random # reference points, no model +python examples/jspaint_circle.py --jspaint ../jspaint --episodes 5 --revision # + thaitea/laya-vision +``` + +JSPaint needs no build step. `JSPaintServer` serves the checkout as static files, and Playwright drives it in +headless Chromium. Set `LAYA_CHROMIUM` (or pass `--chromium`) to use an existing Chromium binary instead of +running `playwright install`. + +Each run writes a new directory under `results/jspaint/`. It contains: + +- `summary.json`: per-policy mean score, pass rate, and the verifier components for every episode; +- one folder per policy and episode, each holding: + - `trajectory.jsonl`: action, cursor, pen state, and the model's probabilities over the actions; + - `final.png`: the last frame; + - `episode.gif`: every frame of the episode. + +## Observation and actions + +Each episode starts the same way. The canvas is cleared to white, the Brush tool is selected (4 px strokes), and +the cursor is placed at a seeded point in the middle of the canvas. + +| | | +|---|---| +| Observation | A screenshot of the 683×384 canvas with the cursor drawn on it: a red ring and cross when the button is up, a filled red dot when it is held down. The `note` text gives the task, the pen state and the step count. | +| Actions (`choice`) | `UP` `DOWN` `LEFT` `RIGHT` `UP_LEFT` `UP_RIGHT` `DOWN_LEFT` `DOWN_RIGHT` each move 24 px (`step_px`) and draw while the pen is down. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | +| End | The episode ends on `DONE` or after 80 steps (`max_steps`). The pen is released automatically at the end. | +| Reward | The verifier score, paid at the end. With `reward="shaped"`, each step instead earns the change in score. | + +The model can only pick from a list of options, so the actions are relative moves. The screenshot has no OS +cursor, so the cursor is drawn onto it. This is the same approach as Maze and Snake. + +The discrete actions are built on a mouse-only tool API: `move_mouse(dx, dy)`, `mouse_down()`, `mouse_up()` and +`screenshot()`. `paintenv.TOOLS` describes these tools as JSON-schema tool definitions, and +`JSPaintEnv.call_tool(name, args)` runs them. A tool-calling agent can therefore drive the same environment with +free coordinates. + +## The circle verifier + +`score_circle(pixels)` reads the canvas pixels from the page with `toDataURL`, so the cursor overlay is never +counted as ink. It fits a least-squares circle to the ink and computes five components, each between 0 and 1: + +- **fit**: how tightly the ink hugs the fitted circle, `1 - rms(|d - r|) / (0.2 r)`; +- **coverage**: the share of 36 angular sectors that contain ink on the ring; +- **clean**: the share of ink within `0.2 r` of the ring, so scribbles and fills count against the drawing; +- **closure**: 1 if no gap in the ring is wider than 30°, falling to 0 at a 120° gap; +- **size**: 1 if the radius is between 8% and 50% of the canvas height and the centre is on the canvas, else 0. + +`score` is the product of the five. An episode `passed` if the score is at least 0.6 and the circle is closed. + +`tests/test_circle_verifier.py` checks both directions: + +- A drawn ring scores 0.95, and an octagon (what the eight moves can draw) scores 0.87. Both pass. +- These all fail: a line, a dot, a tiny ring, a half arc, a three-quarter arc, a square, a flat ellipse, a filled + disc, a ring with a stray line, and random scribbles. + +## Reference points + +`circle_expert()` is a scripted policy. It walks with the pen up to the edge of a circle centred on the canvas, +presses the button, and traces one full turn with the eight moves. `random_policy` picks actions uniformly. On seeds +900000–900002: + +| Policy | Mean score | Pass rate | Mean steps | +|---|---|---|---| +| Scripted expert | 0.76 | 3/3 | 37.7 | +| Random | 0.00 | 0/3 | 8.7 | +| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 2.0 | + +The zero-shot checkpoint's action probabilities are nearly uniform, 0.10–0.15 for the top action against 1/11 ≈ +0.09. It chooses `DONE` within 1–3 steps. The model was never trained on this task. The expert's trajectories +(screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. diff --git a/tests/test_circle_verifier.py b/tests/test_circle_verifier.py new file mode 100644 index 00000000..0f33a270 --- /dev/null +++ b/tests/test_circle_verifier.py @@ -0,0 +1,74 @@ +"""The circle verifier passes circles (including the rough polygon the eight-move action set draws) and fails the +shapes a policy might draw instead.""" +import math +import random + +import numpy as np +import pytest + +from laya.circle_verifier import fit_circle, score_circle + +PIL = pytest.importorskip("PIL") +from PIL import Image, ImageDraw # noqa: E402 + +W, H = 683, 384 # JSPaint's default canvas + + +def canvas(draw): + im = Image.new("RGB", (W, H), "white") + draw(ImageDraw.Draw(im)) + return np.asarray(im) + + +def polygon(n, r=110, cx=340, cy=190, rot=math.pi / 8): + return [(cx + r * math.cos(rot + 2 * math.pi * i / n), cy + r * math.sin(rot + 2 * math.pi * i / n)) + for i in range(n + 1)] + + +def test_fit_circle_recovers_centre_and_radius(): + t = np.linspace(0, 2 * np.pi, 200) + cx, cy, r = fit_circle(300 + 80 * np.cos(t), 150 + 80 * np.sin(t)) + assert (round(cx), round(cy), round(r)) == (300, 150, 80) + + +@pytest.mark.parametrize("name,draw", [ + ("ring", lambda d: d.ellipse([230, 80, 450, 300], outline="black", width=4)), + ("octagon", lambda d: d.line(polygon(8), fill="black", width=4)), + ("coloured ring", lambda d: d.ellipse([100, 50, 300, 250], outline=(200, 0, 0), width=6)), +]) +def test_circles_pass(name, draw): + s = score_circle(canvas(draw)) + assert s["passed"], (name, s) + assert s["score"] >= 0.8 and s["coverage"] == 1.0 + + +@pytest.mark.parametrize("name,draw", [ + ("blank", lambda d: None), + ("line", lambda d: d.line([100, 100, 500, 300], fill="black", width=4)), + ("dot", lambda d: d.ellipse([300, 150, 310, 160], fill="black")), + ("tiny ring", lambda d: d.ellipse([300, 150, 330, 180], outline="black", width=4)), + ("half arc", lambda d: d.arc([230, 80, 450, 300], 0, 180, fill="black", width=4)), + ("three-quarter arc", lambda d: d.arc([230, 80, 450, 300], 0, 270, fill="black", width=4)), + ("square", lambda d: d.rectangle([230, 80, 450, 300], outline="black", width=4)), + ("flat ellipse", lambda d: d.ellipse([140, 120, 540, 270], outline="black", width=4)), + ("filled disc", lambda d: d.ellipse([230, 80, 450, 300], fill="black")), + ("ring plus stray line", lambda d: (d.ellipse([230, 80, 450, 300], outline="black", width=4), + d.line([0, 20, 680, 20], fill="black", width=4))), +]) +def test_non_circles_fail(name, draw): + s = score_circle(canvas(draw)) + assert not s["passed"], (name, s) + assert s["score"] < 0.6 + + +def test_random_scribbles_fail(): + for seed in range(5): + rng = random.Random(seed) + pts = [(rng.uniform(0, W), rng.uniform(0, H)) for _ in range(15)] + assert not score_circle(canvas(lambda d: d.line(pts, fill="black", width=4)))["passed"] + + +def test_arc_scores_below_closed_circle(): + closed = score_circle(canvas(lambda d: d.ellipse([230, 80, 450, 300], outline="black", width=4))) + arc = score_circle(canvas(lambda d: d.arc([230, 80, 450, 300], 0, 270, fill="black", width=4))) + assert arc["score"] < closed["score"] and arc["max_gap_deg"] > 30 diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py new file mode 100644 index 00000000..8d2c04bc --- /dev/null +++ b/tests/test_paintenv.py @@ -0,0 +1,92 @@ +"""The JSPaint environment end to end in headless Chromium: mouse actions reach the real canvas, and the scripted +circle expert passes the verifier. Skipped without Playwright, a Chromium, or a JSPaint checkout (JSPAINT_DIR, default +a sibling of this repo).""" +import os + +import numpy as np +import pytest + +pytest.importorskip("playwright") +pytest.importorskip("PIL") + +from laya.circle_verifier import ink_mask # noqa: E402 +from laya.games import paint_question # noqa: E402 +from laya.paintenv import ACTIONS, TOOLS, JSPaintEnv, JSPaintServer, circle_expert, play_episodes, \ + random_policy # noqa: E402 + +JSPAINT = os.environ.get("JSPAINT_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "jspaint")) +if not os.path.exists(os.path.join(JSPAINT, "index.html")): + pytest.skip("no JSPaint checkout at %s" % JSPAINT, allow_module_level=True) + + +@pytest.fixture(scope="module") +def env(): + server = JSPaintServer(JSPAINT) + try: + e = JSPaintEnv(server.url, max_steps=80) + except Exception as err: # no Chromium installed + server.close() + pytest.skip("cannot launch Chromium: %s" % err) + yield e + e.close() + server.close() + + +def test_question_covers_actions_and_tools_are_mouse_only(): + assert set(paint_question()["action"]["criteria"]) == set(ACTIONS) + assert {t["name"] for t in TOOLS} == {"move_mouse", "mouse_down", "mouse_up", "screenshot"} + + +def test_reset_is_blank_and_observation_is_canvas(env): + obs = env.reset(seed=3) + assert obs.size == (env.width, env.height) + assert not ink_mask(env.canvas_pixels()).any() + assert not env.done and not env.pen + + +def test_pen_down_moves_draw_and_pen_up_moves_do_not(env): + env.reset(seed=0) + x0, y0 = env.cursor + env.step("PEN_DOWN") + for _ in range(4): + env.step("RIGHT") + env.step("PEN_UP") + for _ in range(3): + env.step("DOWN") + ink = ink_mask(env.canvas_pixels()) + ys, xs = np.nonzero(ink) + assert len(xs) > 50 + assert abs(np.median(ys) - y0) <= 3 # one horizontal stroke on the starting row + assert xs.min() >= x0 - 5 and xs.max() <= x0 + 4 * env.step_px + 5 + assert env.cursor == (x0 + 4 * env.step_px, y0 + 3 * env.step_px) + + +def test_cursor_is_clamped_to_canvas(env): + env.reset(seed=0) + for _ in range(30): + env.step("UP_LEFT") + assert env.cursor == (1.0, 1.0) + + +def test_done_ends_episode_with_verifier_result(env): + env.reset(seed=0) + _, reward, done, info = env.step("DONE") + assert done and reward == 0.0 and info["verifier"]["score"] == 0.0 + with pytest.raises(RuntimeError): + env.step("UP") + + +def test_call_tool_draws(env): + env.reset(seed=1) + env.call_tool("mouse_down") + env.call_tool("move_mouse", {"dx": 60, "dy": 30}) + env.call_tool("mouse_up") + assert ink_mask(env.canvas_pixels()).sum() > 50 + assert env.call_tool("screenshot").size == (env.width, env.height) + + +def test_expert_passes_and_random_does_not(env): + expert = play_episodes(env, circle_expert(), episodes=2, seed=0) + assert expert["pass_rate"] == 1.0 and expert["mean_score"] >= 0.7 + rnd = play_episodes(env, random_policy(0), episodes=3, seed=0) + assert rnd["pass_rate"] == 0.0 and rnd["mean_score"] < expert["mean_score"] From 02c9e268e0d1d29204c68bff718e29c22ac7247e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 19:21:09 +0000 Subject: [PATCH 02/24] JSPaint env: 16-direction 8 px moves and a roundness-based circle verifier Eight 24 px moves could only draw an octagon. The action set is now a move toward each of 16 compass points (every 22.5 degrees) of step_px (default 8, max_steps 200); directions=8 and step_px stay configurable, and paint_question builds the options from them. The verifier now scores roundness as the spread of per-sector mean radii, which cancels stroke width and measures the shape itself: a ring scores 0.99, a 16-gon 0.88, an octagon 0.56 (now failing), with the pass bar at 0.7. The scripted expert takes the forward move that stays closest to the radius and scores 0.91 (was 0.76 as an octagon). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- examples/jspaint_circle.py | 16 +++--- laya/circle_verifier.py | 35 +++++++----- laya/games.py | 24 +++++--- laya/paintenv.py | 97 ++++++++++++++++++++++----------- site-docs/how-to/jspaint-env.md | 32 ++++++----- tests/test_circle_verifier.py | 12 ++-- tests/test_paintenv.py | 22 +++++--- 7 files changed, 149 insertions(+), 89 deletions(-) diff --git a/examples/jspaint_circle.py b/examples/jspaint_circle.py index 7a043047..1a49deaa 100644 --- a/examples/jspaint_circle.py +++ b/examples/jspaint_circle.py @@ -1,10 +1,10 @@ """Ask Laya Vision to draw a circle in JSPaint with the mouse only, and score it. Each step a screenshot of the JSPaint canvas (with the cursor marked on it) goes into the model as the image, and a -`choice` question over mouse actions (eight moves, pen down, pen up, done) picks the next move, which is sent to the -real app as pointer events. At the end the circle verifier scores the true canvas pixels. The scripted expert and -a random policy play the same seeds as reference points, and the model's score is also reported normalized, -(model - random) / (expert - random), as in the games suite. +`choice` question over mouse actions (an 8 px move toward each of 16 compass directions, pen down, pen up, done) +picks the next move, which is sent to the real app as pointer events. At the end the circle verifier scores the +true canvas pixels. The scripted expert and a random policy play the same seeds as reference points, and the +model's score is also reported normalized, (model - random) / (expert - random), as in the games suite. pip install -e . playwright pillow git clone https://github.com/r33drichards/jspaint ../jspaint @@ -31,8 +31,9 @@ def main(): ap.add_argument("--policies", default="model,expert,random", help="comma list of model, expert, random") ap.add_argument("--episodes", type=int, default=5) ap.add_argument("--seed", type=int, default=900_000) - ap.add_argument("--max-steps", type=int, default=80) - ap.add_argument("--step-px", type=int, default=24) + ap.add_argument("--max-steps", type=int, default=200) + ap.add_argument("--step-px", type=int, default=8) + ap.add_argument("--directions", type=int, default=16, choices=(8, 16)) ap.add_argument("--model", default="thaitea/laya-vision") ap.add_argument("--revision", default=None, help="pin the Hub revision of --model") ap.add_argument("--device", default=None) @@ -47,7 +48,8 @@ def main(): summary = {"task": "circle", "jspaint": os.path.abspath(args.jspaint), "args": vars(args), "policies": {}} with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, max_steps=args.max_steps, - step_px=args.step_px, headless=not args.headed, + step_px=args.step_px, directions=args.directions, + headless=not args.headed, executable_path=args.chromium, keep_frames=True) as env: for name in names: if name == "model": diff --git a/laya/circle_verifier.py b/laya/circle_verifier.py index b7a4a993..9ff1af4a 100644 --- a/laya/circle_verifier.py +++ b/laya/circle_verifier.py @@ -8,8 +8,10 @@ the centre ``(cx, cy)`` and radius ``r``. 3. **Components**, each in [0, 1]: - * ``fit``: how tightly the ink hugs the fitted circle, ``1 - rms(|d - r|) / (fit_tol * r)``, where ``d`` is each - ink pixel's distance from the centre; + * ``roundness``: how constant the radius is around the ring, ``1 - std(r_k) / (round_tol * r)``, where ``r_k`` is + the mean distance from the centre of the ring ink in angular sector ``k``. Averaging within a sector cancels the + stroke width, so this measures the shape itself: a clean ring scores ~1, a 16-sided walk ~0.9, an octagon ~0.5, + a square or a 4:3 ellipse 0; * ``coverage``: the fraction of ``bins`` equal angular sectors around the centre that hold ink on the ring, so an arc or a C-shape loses the missing sectors; * ``clean``: the fraction of ink on the ring band (within ``band * r`` of it), so scribbles, fills and stray @@ -19,12 +21,12 @@ * ``size``: 1 when ``r`` is between ``min_radius`` and ``max_radius`` of the shorter canvas side and the centre is on the canvas, else 0 (a dot or a straight line fits a tiny or enormous "circle"). -4. ``score = size * fit * coverage * clean * closure``. ``passed`` needs ``score >= pass_score`` and no angular gap - wider than ``max_gap_deg`` (the circle is closed). +4. ``score = size * roundness * coverage * clean * closure``. ``passed`` needs ``score >= pass_score`` and no + angular gap wider than ``max_gap_deg`` (the circle is closed). -The tolerances are loose enough that a circle walked with only eight fixed-length moves (a rough polygon, which is -what the environment's action set can draw) still passes, and tight enough that lines, arcs, squares, filled discs, -dots and scribbles fail; ``tests/test_circle_verifier.py`` pins both sides. +The tolerances pass a circle walked with the environment's 16 fine moves and fail an octagon, lines, arcs, squares, +ellipses, filled discs, dots and scribbles; ``tests/test_circle_verifier.py`` pins both sides. ``rms`` (the RMS +distance of all ink from the fitted circle, over ``r``) is reported alongside for diagnosis. """ from typing import Dict, Sequence @@ -63,15 +65,15 @@ def _max_gap_deg(occupied: np.ndarray) -> float: def score_circle(pixels, bg: Sequence[int] = (255, 255, 255), ink_threshold: int = 60, min_ink: int = 30, - fit_tol: float = 0.2, band: float = 0.2, bins: int = 36, min_radius: float = 0.08, - max_radius: float = 0.5, max_gap_deg: float = 30.0, pass_score: float = 0.6) -> Dict: + round_tol: float = 0.05, band: float = 0.2, bins: int = 36, min_radius: float = 0.08, + max_radius: float = 0.5, max_gap_deg: float = 30.0, pass_score: float = 0.7) -> Dict: """Score an ``(h, w, 3)`` canvas for one drawn circle. Returns ``score`` in [0, 1], ``passed``, every component, and the fitted ``cx, cy, r`` (see the module docstring).""" mask = ink_mask(pixels, bg, ink_threshold) h, w = mask.shape ys, xs = np.nonzero(mask) - out = {"score": 0.0, "passed": False, "ink": int(len(xs)), "fit": 0.0, "coverage": 0.0, "clean": 0.0, - "closure": 0.0, "size": 0.0, "max_gap_deg": 360.0, "cx": None, "cy": None, "r": None} + out = {"score": 0.0, "passed": False, "ink": int(len(xs)), "roundness": 0.0, "rms": None, "coverage": 0.0, + "clean": 0.0, "closure": 0.0, "size": 0.0, "max_gap_deg": 360.0, "cx": None, "cy": None, "r": None} if len(xs) < min_ink: return out cx, cy, r = fit_circle(xs, ys) @@ -82,19 +84,22 @@ def score_circle(pixels, bg: Sequence[int] = (255, 255, 255), ink_threshold: int return out d = np.hypot(xs - cx, ys - cy) dev = np.abs(d - r) - fit = float(np.clip(1.0 - np.sqrt(np.mean(dev ** 2)) / (fit_tol * r), 0.0, 1.0)) on_ring = dev <= max(band * r, 3.0) clean = float(on_ring.mean()) ang = np.arctan2(ys[on_ring] - cy, xs[on_ring] - cx) idx = ((ang + np.pi) / (2 * np.pi) * bins).astype(int) % bins occupied = np.zeros(bins, dtype=bool) occupied[idx] = True + counts = np.bincount(idx, minlength=bins)[occupied] + radii = np.bincount(idx, weights=d[on_ring], minlength=bins)[occupied] / counts + roundness = float(np.clip(1.0 - np.std(radii) / (round_tol * r), 0.0, 1.0)) coverage = float(occupied.mean()) gap = _max_gap_deg(occupied) closure = float(np.clip(1.0 - (gap - max_gap_deg) / 90.0, 0.0, 1.0)) - score = float(size_ok) * fit * coverage * clean * closure - out.update(score=round(score, 4), fit=round(fit, 4), coverage=round(coverage, 4), clean=round(clean, 4), - closure=round(closure, 4), size=float(size_ok), max_gap_deg=round(gap, 1), + score = float(size_ok) * roundness * coverage * clean * closure + rms = float(np.sqrt(np.mean(dev ** 2)) / r) + out.update(score=round(score, 4), roundness=round(roundness, 4), rms=round(rms, 4), coverage=round(coverage, 4), + clean=round(clean, 4), closure=round(closure, 4), size=float(size_ok), max_gap_deg=round(gap, 1), passed=bool(score >= pass_score and gap <= max_gap_deg)) return out diff --git a/laya/games.py b/laya/games.py index 2592adca..2b6b3dd2 100644 --- a/laya/games.py +++ b/laya/games.py @@ -77,27 +77,33 @@ def snake_question() -> Dict: }} -PAINT_ACTIONS = { - "UP": "move the cursor up", "DOWN": "move the cursor down", "LEFT": "move the cursor left", - "RIGHT": "move the cursor right", "UP_LEFT": "move the cursor diagonally up and to the left", - "UP_RIGHT": "move the cursor diagonally up and to the right", - "DOWN_LEFT": "move the cursor diagonally down and to the left", - "DOWN_RIGHT": "move the cursor diagonally down and to the right", +PAINT_DIRECTIONS = { + "N": "straight up", "NNE": "up and slightly right", "NE": "diagonally up and right", + "ENE": "right and slightly up", "E": "straight right", "ESE": "right and slightly down", + "SE": "diagonally down and right", "SSE": "down and slightly right", "S": "straight down", + "SSW": "down and slightly left", "SW": "diagonally down and left", "WSW": "left and slightly down", + "W": "straight left", "WNW": "left and slightly up", "NW": "diagonally up and left", "NNW": "up and slightly left", +} +PAINT_PEN = { "PEN_DOWN": "press the mouse button to start drawing", "PEN_UP": "release the mouse button to stop drawing", "DONE": "the drawing is finished", } PAINT_GOALS = {"circle": "draw one round, closed circle, about as big as a third of the canvas height"} -def paint_question(task: str = "circle") -> Dict: - """The question for ``laya.paintenv.JSPaintEnv``: a paint canvas, a red cursor, and mouse-only actions.""" +def paint_question(task: str = "circle", directions: int = 16, step_px: int = 8) -> Dict: + """The question for ``laya.paintenv.JSPaintEnv``: a paint canvas, a red cursor, and mouse-only actions (a + ``step_px`` move toward each of ``directions`` compass points, then pen down / pen up / done).""" + from laya.paintenv import compass_moves + + moves = {m: "move the cursor %d pixels %s" % (step_px, PAINT_DIRECTIONS[m]) for m in compass_moves(directions)} return {"action": { "type": "choice", "instructions": "You are using a paint program with only the mouse. The white area is the canvas and the " "red mark is the mouse cursor: a hollow ring with a cross means the button is up, a filled " "dot means it is held down and moving draws a black line. Your task: %s. Which mouse action " "should you take now?" % PAINT_GOALS[task], - "criteria": dict(PAINT_ACTIONS), + "criteria": {**moves, **PAINT_PEN}, }} diff --git a/laya/paintenv.py b/laya/paintenv.py index 37457798..8d9976d1 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -8,8 +8,10 @@ * **observation** is a screenshot of the canvas with the cursor drawn on it (headless screenshots have no OS cursor): a red ring when the pen is up, a filled red dot when it is down. ``note()`` carries the pen state and step count as text; -* **actions** are mouse-only. ``ACTIONS`` is the discrete set the image model chooses from (eight ``step_px`` moves, - ``PEN_DOWN``, ``PEN_UP``, ``DONE``); each maps onto the tool API ``move_mouse(dx, dy)`` / ``mouse_down()`` / +* **actions** are mouse-only. ``env.actions`` is the discrete set the image model chooses from: a ``step_px`` move + toward each of ``directions`` compass points (16 by default: ``N``, ``NNE``, ``NE``, ... every 22.5 degrees, 8 px, + fine enough to walk a round circle; 8 gives the coarse ``N NE E ...`` set), plus ``PEN_DOWN``, ``PEN_UP`` and + ``DONE``. Each maps onto the tool API ``move_mouse(dx, dy)`` / ``mouse_down()`` / ``mouse_up()``, which sends real pointer events to the page. ``TOOLS`` describes that API as JSON-schema tools, so a tool-calling agent can drive the same environment with ``call_tool``; * **reward** is the task verifier on the true canvas pixels (``canvas_pixels()``, read from the page, so the cursor @@ -17,7 +19,7 @@ the episode, or as the per-step change in score with ``reward="shaped"``. An episode ends on ``DONE`` or after ``max_steps``. -``circle_expert`` (a scripted policy that walks a circle with the same eight moves), ``random_policy`` and +``circle_expert`` (a scripted policy that walks a circle with the same moves), ``random_policy`` and ``model_policy`` (``VLMAgent.predict`` on the screenshot) are the reference points; ``play_episodes`` runs any of them. ``examples/jspaint_circle.py`` is the command-line runner. @@ -37,11 +39,21 @@ import numpy as np -MOVES = { - "UP": (0, -1), "DOWN": (0, 1), "LEFT": (-1, 0), "RIGHT": (1, 0), - "UP_LEFT": (-1, -1), "UP_RIGHT": (1, -1), "DOWN_LEFT": (-1, 1), "DOWN_RIGHT": (1, 1), -} -ACTIONS = tuple(MOVES) + ("PEN_DOWN", "PEN_UP", "DONE") +COMPASS = ("N", "NNE", "NE", "ENE", "E", "ESE", "SE", "SSE", "S", "SSW", "SW", "WSW", "W", "WNW", "NW", "NNW") +PEN_ACTIONS = ("PEN_DOWN", "PEN_UP", "DONE") + + +def compass_moves(directions: int = 16) -> Dict[str, Tuple[float, float]]: + """``{name: (ux, uy)}`` unit vectors (screen y points down) for 8 or 16 compass directions, clockwise from ``N``.""" + if directions not in (8, 16): + raise ValueError("directions must be 8 or 16") + every = 16 // directions + return {COMPASS[i]: (round(math.sin(math.radians(22.5 * i)), 6), round(-math.cos(math.radians(22.5 * i)), 6)) + for i in range(0, 16, every)} + + +MOVES = compass_moves(16) +ACTIONS = tuple(MOVES) + PEN_ACTIONS TASKS = {"circle": "Draw a circle on the canvas."} TOOLS = [ @@ -93,9 +105,9 @@ def __exit__(self, *exc): class JSPaintEnv: - """One headless JSPaint page; ``reset`` starts an episode and ``step(action)`` plays one of ``ACTIONS``.""" + """One headless JSPaint page; ``reset`` starts an episode and ``step(action)`` plays one of ``env.actions``.""" - def __init__(self, url: str, task: str = "circle", step_px: int = 24, max_steps: int = 80, + def __init__(self, url: str, task: str = "circle", step_px: int = 8, directions: int = 16, max_steps: int = 200, reward: str = "terminal", headless: bool = True, executable_path: Optional[str] = None, viewport: Tuple[int, int] = (800, 600), keep_frames: bool = False): if task not in TASKS: @@ -105,7 +117,9 @@ def __init__(self, url: str, task: str = "circle", step_px: int = 24, max_steps: from playwright.sync_api import sync_playwright self.url, self.task, self.step_px, self.max_steps, self.reward_mode = url, task, step_px, max_steps, reward - self.keep_frames = keep_frames + self.keep_frames, self.directions = keep_frames, directions + self.moves = compass_moves(directions) + self.actions = tuple(self.moves) + PEN_ACTIONS self._pw = sync_playwright().start() self.browser = self._pw.chromium.launch(headless=headless, executable_path=executable_path or default_chromium()) @@ -148,14 +162,14 @@ def done(self) -> bool: return self.finished or self.steps >= self.max_steps def step(self, action: str): - """Play one of ``ACTIONS``. Returns ``(observation, reward, done, info)``; ``info["verifier"]`` holds the full - verifier result once the episode is over.""" - if action not in ACTIONS: + """Play one of ``env.actions``. Returns ``(observation, reward, done, info)``; ``info["verifier"]`` holds the + full verifier result once the episode is over.""" + if action not in self.actions: raise ValueError("unknown action %r" % action) if self.done: raise RuntimeError("episode is over; call reset()") - if action in MOVES: - dx, dy = MOVES[action] + if action in self.moves: + dx, dy = self.moves[action] self.move_mouse(dx * self.step_px, dy * self.step_px) elif action == "PEN_DOWN": self.mouse_down() @@ -195,7 +209,8 @@ def move_mouse(self, dx: float, dy: float) -> None: x = min(max(self.cursor[0] + dx, 1.0), self.width - 2.0) y = min(max(self.cursor[1] + dy, 1.0), self.height - 2.0) self.cursor = (x, y) - self.page.mouse.move(*self._client(self.cursor), steps=4 if self.pen else 1) + steps = max(2, int(math.hypot(dx, dy) // 4)) if self.pen else 1 + self.page.mouse.move(*self._client(self.cursor), steps=steps) def mouse_down(self) -> None: if not self.pen: @@ -271,16 +286,30 @@ def __exit__(self, *exc): # -- policies ------------------------------------------------------------------------------------------------------ -def _toward(pos, target) -> str: +def _toward(moves, pos, target) -> str: """The move whose unit direction best matches ``target - pos``.""" vx, vy = target[0] - pos[0], target[1] - pos[1] - return max(MOVES, key=lambda a: (MOVES[a][0] * vx + MOVES[a][1] * vy) / math.hypot(*MOVES[a])) + return max(moves, key=lambda a: moves[a][0] * vx + moves[a][1] * vy) + + +def _around(moves, pos, centre, r, step) -> str: + """Of the moves that advance clockwise round ``centre`` by at least a third of a step, the one that lands + closest to radius ``r`` (ties to the larger advance).""" + ang = math.atan2(pos[1] - centre[1], pos[0] - centre[0]) + + def outcome(a): + x, y = pos[0] + moves[a][0] * step, pos[1] + moves[a][1] * step + adv = (math.atan2(y - centre[1], x - centre[0]) - ang + math.pi) % (2 * math.pi) - math.pi + return abs(math.hypot(x - centre[0], y - centre[1]) - r), -adv + + forward = [a for a in moves if -outcome(a)[1] >= step / r / 3] or list(moves) + return min(forward, key=outcome) def circle_expert(radius_frac: float = 0.3): """A scripted policy: walk (pen up) to the rightmost point of a circle centred on the canvas, press, trace one - full turn plus a little overlap with the eight moves (each step heads for the point one step further round the - circle), release, ``DONE``. Deterministic given the environment state.""" + full turn plus a little overlap (each step takes the forward move that stays closest to the radius), release, + ``DONE``. Deterministic given the environment state.""" state: Dict = {} def policy(env: JSPaintEnv) -> str: @@ -294,7 +323,7 @@ def policy(env: JSPaintEnv) -> str: if state["phase"] == "approach": start = (cx + r, cy) if math.dist(pos, start) > step * 0.75: - return _toward(pos, start) + return _toward(env.moves, pos, start) state["phase"] = "trace" state["prev"] = math.atan2(pos[1] - cy, pos[0] - cx) return "PEN_DOWN" @@ -304,8 +333,7 @@ def policy(env: JSPaintEnv) -> str: state["turned"] += delta state["prev"] = ang if state["turned"] < 2 * math.pi + 0.3: - nxt = ang + step / r - return _toward(pos, (cx + r * math.cos(nxt), cy + r * math.sin(nxt))) + return _around(env.moves, pos, (cx, cy), r, step) state["phase"] = "lift" return "PEN_UP" return "DONE" @@ -315,20 +343,23 @@ def policy(env: JSPaintEnv) -> str: def random_policy(seed: int = 0): rng = random.Random(seed) - return lambda env: rng.choice(ACTIONS) + return lambda env: rng.choice(env.actions) class ModelPolicy: """The model's most likely action from the screenshot plus the text note, via ``predict``. ``last`` keeps the - latest answer (probabilities over ``ACTIONS``) for logging.""" + latest answer (probabilities over ``env.actions``) for logging.""" def __init__(self, agent, task: str = "circle"): - from laya.games import paint_question - - self.agent, self.question, self.last, self.provenance = agent, paint_question(task), None, None + self.agent, self.task, self.last, self.provenance, self._questions = agent, task, None, None, {} def __call__(self, env: JSPaintEnv) -> str: - out = self.agent.predict({"image": env.render(), "note": env.note()}, self.question) + from laya.games import paint_question + + key = (env.directions, env.step_px) + if key not in self._questions: + self._questions[key] = paint_question(self.task, env.directions, env.step_px) + out = self.agent.predict({"image": env.render(), "note": env.note()}, self._questions[key]) self.last, self.provenance = out["answers"]["action"], out.get("provenance") return self.last["choice"] @@ -352,11 +383,13 @@ def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step eps.append({"seed": seed + i, "steps": env.steps, "ended": "done" if env.finished else "capped", **env.result}) scores = [e["score"] for e in eps] - return {"task": env.task, "episodes": episodes, "seed": seed, "step_px": env.step_px, "max_steps": env.max_steps, + return {"task": env.task, "episodes": episodes, "seed": seed, "directions": env.directions, + "step_px": env.step_px, "max_steps": env.max_steps, "mean_score": float(np.mean(scores)), "median_score": float(np.median(scores)), "pass_rate": float(np.mean([e["passed"] for e in eps])), "mean_steps": float(np.mean([e["steps"] for e in eps])), "actions": dict(counts), "results": eps} -__all__ = ["ACTIONS", "MOVES", "TASKS", "TOOLS", "JSPaintServer", "JSPaintEnv", "circle_expert", "random_policy", +__all__ = ["ACTIONS", "MOVES", "COMPASS", "PEN_ACTIONS", "compass_moves", "TASKS", "TOOLS", "JSPaintServer", + "JSPaintEnv", "circle_expert", "random_policy", "ModelPolicy", "model_policy", "play_episodes", "default_chromium"] diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index 5f8a53e7..718656bc 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -37,11 +37,14 @@ the cursor is placed at a seeded point in the middle of the canvas. | | | |---|---| | Observation | A screenshot of the 683×384 canvas with the cursor drawn on it: a red ring and cross when the button is up, a filled red dot when it is held down. The `note` text gives the task, the pen state and the step count. | -| Actions (`choice`) | `UP` `DOWN` `LEFT` `RIGHT` `UP_LEFT` `UP_RIGHT` `DOWN_LEFT` `DOWN_RIGHT` each move 24 px (`step_px`) and draw while the pen is down. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | -| End | The episode ends on `DONE` or after 80 steps (`max_steps`). The pen is released automatically at the end. | +| Actions (`choice`) | 16 compass moves, `N` `NNE` `NE` `ENE` `E` … `NNW`, one every 22.5°. Each moves 8 px (`step_px`) and draws while the pen is down. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | +| End | The episode ends on `DONE` or after 200 steps (`max_steps`). The pen is released automatically at the end. | | Reward | The verifier score, paid at the end. With `reward="shaped"`, each step instead earns the change in score. | -The model can only pick from a list of options, so the actions are relative moves. The screenshot has no OS +The model can only pick from a list of options, so the actions are relative moves. They are fine-grained because +coarse moves can't draw a circle. With 8 directions and 24 px steps, the best walk is an octagon, which scores 0.39 +and fails. With 16 directions, 12 px steps score 0.83 and 8 px steps score 0.91. Both `directions=8|16` and +`step_px` are settable (`--directions`, `--step-px`). Smaller steps draw rounder circles but take more steps. The screenshot has no OS cursor, so the cursor is drawn onto it. This is the same approach as Maze and Snake. The discrete actions are built on a mouse-only tool API: `move_mouse(dx, dy)`, `mouse_down()`, `mouse_up()` and @@ -54,32 +57,35 @@ free coordinates. `score_circle(pixels)` reads the canvas pixels from the page with `toDataURL`, so the cursor overlay is never counted as ink. It fits a least-squares circle to the ink and computes five components, each between 0 and 1: -- **fit**: how tightly the ink hugs the fitted circle, `1 - rms(|d - r|) / (0.2 r)`; +- **roundness**: how constant the radius is around the ring. It is `1 - std(r_k) / (0.05 r)`, where `r_k` is the + mean radius of the ink in each of 36 angular sectors. Averaging within a sector cancels the stroke width, so this + measures the shape itself: a clean ring scores 0.99, a 16-sided polygon 0.88, an octagon 0.56, and a square or a + 4:3 ellipse 0; - **coverage**: the share of 36 angular sectors that contain ink on the ring; - **clean**: the share of ink within `0.2 r` of the ring, so scribbles and fills count against the drawing; - **closure**: 1 if no gap in the ring is wider than 30°, falling to 0 at a 120° gap; - **size**: 1 if the radius is between 8% and 50% of the canvas height and the centre is on the canvas, else 0. -`score` is the product of the five. An episode `passed` if the score is at least 0.6 and the circle is closed. +`score` is the product of the five. An episode `passed` if the score is at least 0.7 and the circle is closed. `tests/test_circle_verifier.py` checks both directions: -- A drawn ring scores 0.95, and an octagon (what the eight moves can draw) scores 0.87. Both pass. -- These all fail: a line, a dot, a tiny ring, a half arc, a three-quarter arc, a square, a flat ellipse, a filled +- These pass: a drawn ring, a small ring, a coloured ring, and a 16-sided polygon (what the fine moves draw). +- These all fail: an octagon, a line, a dot, a tiny ring, a half arc, a three-quarter arc, a square, a flat ellipse, a filled disc, a ring with a stray line, and random scribbles. ## Reference points `circle_expert()` is a scripted policy. It walks with the pen up to the edge of a circle centred on the canvas, -presses the button, and traces one full turn with the eight moves. `random_policy` picks actions uniformly. On seeds +presses the button, and traces one full turn. Each step it takes the forward move that lands closest to the radius. `random_policy` picks actions uniformly. On seeds 900000–900002: | Policy | Mean score | Pass rate | Mean steps | |---|---|---|---| -| Scripted expert | 0.76 | 3/3 | 37.7 | -| Random | 0.00 | 0/3 | 8.7 | -| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 2.0 | +| Scripted expert | 0.91 | 3/3 | 116.3 | +| Random | 0.00 | 0/3 | 10.7 | +| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 167.3 | -The zero-shot checkpoint's action probabilities are nearly uniform, 0.10–0.15 for the top action against 1/11 ≈ -0.09. It chooses `DONE` within 1–3 steps. The model was never trained on this task. The expert's trajectories +The zero-shot checkpoint almost always picks `PEN_UP` (496 of 502 actions), which does nothing while the pen is +already up. It runs out the step limit without drawing anything. The model was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. diff --git a/tests/test_circle_verifier.py b/tests/test_circle_verifier.py index 0f33a270..bdd18d64 100644 --- a/tests/test_circle_verifier.py +++ b/tests/test_circle_verifier.py @@ -1,5 +1,5 @@ -"""The circle verifier passes circles (including the rough polygon the eight-move action set draws) and fails the -shapes a policy might draw instead.""" +"""The circle verifier passes round circles (including the 16-sided walk the environment's fine moves draw) and fails +the shapes a policy might draw instead, including an octagon (what coarse eight-direction moves draw).""" import math import random @@ -33,13 +33,14 @@ def test_fit_circle_recovers_centre_and_radius(): @pytest.mark.parametrize("name,draw", [ ("ring", lambda d: d.ellipse([230, 80, 450, 300], outline="black", width=4)), - ("octagon", lambda d: d.line(polygon(8), fill="black", width=4)), + ("16-gon", lambda d: d.line(polygon(16), fill="black", width=4)), + ("small ring", lambda d: d.ellipse([305, 155, 375, 225], outline="black", width=4)), ("coloured ring", lambda d: d.ellipse([100, 50, 300, 250], outline=(200, 0, 0), width=6)), ]) def test_circles_pass(name, draw): s = score_circle(canvas(draw)) assert s["passed"], (name, s) - assert s["score"] >= 0.8 and s["coverage"] == 1.0 + assert s["score"] >= 0.85 and s["coverage"] == 1.0 @pytest.mark.parametrize("name,draw", [ @@ -49,6 +50,7 @@ def test_circles_pass(name, draw): ("tiny ring", lambda d: d.ellipse([300, 150, 330, 180], outline="black", width=4)), ("half arc", lambda d: d.arc([230, 80, 450, 300], 0, 180, fill="black", width=4)), ("three-quarter arc", lambda d: d.arc([230, 80, 450, 300], 0, 270, fill="black", width=4)), + ("octagon", lambda d: d.line(polygon(8), fill="black", width=4)), ("square", lambda d: d.rectangle([230, 80, 450, 300], outline="black", width=4)), ("flat ellipse", lambda d: d.ellipse([140, 120, 540, 270], outline="black", width=4)), ("filled disc", lambda d: d.ellipse([230, 80, 450, 300], fill="black")), @@ -58,7 +60,7 @@ def test_circles_pass(name, draw): def test_non_circles_fail(name, draw): s = score_circle(canvas(draw)) assert not s["passed"], (name, s) - assert s["score"] < 0.6 + assert s["score"] < 0.7 def test_random_scribbles_fail(): diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py index 8d2c04bc..07986426 100644 --- a/tests/test_paintenv.py +++ b/tests/test_paintenv.py @@ -11,7 +11,7 @@ from laya.circle_verifier import ink_mask # noqa: E402 from laya.games import paint_question # noqa: E402 -from laya.paintenv import ACTIONS, TOOLS, JSPaintEnv, JSPaintServer, circle_expert, play_episodes, \ +from laya.paintenv import ACTIONS, TOOLS, compass_moves, JSPaintEnv, JSPaintServer, circle_expert, play_episodes, \ random_policy # noqa: E402 JSPAINT = os.environ.get("JSPAINT_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "jspaint")) @@ -23,7 +23,7 @@ def env(): server = JSPaintServer(JSPAINT) try: - e = JSPaintEnv(server.url, max_steps=80) + e = JSPaintEnv(server.url) except Exception as err: # no Chromium installed server.close() pytest.skip("cannot launch Chromium: %s" % err) @@ -34,6 +34,11 @@ def env(): def test_question_covers_actions_and_tools_are_mouse_only(): assert set(paint_question()["action"]["criteria"]) == set(ACTIONS) + assert len(ACTIONS) == 16 + 3 + assert set(paint_question(directions=8)["action"]["criteria"]) == set(compass_moves(8)) | {"PEN_DOWN", "PEN_UP", + "DONE"} + for ux, uy in compass_moves(16).values(): + assert abs(ux * ux + uy * uy - 1) < 1e-5 assert {t["name"] for t in TOOLS} == {"move_mouse", "mouse_down", "mouse_up", "screenshot"} @@ -49,10 +54,10 @@ def test_pen_down_moves_draw_and_pen_up_moves_do_not(env): x0, y0 = env.cursor env.step("PEN_DOWN") for _ in range(4): - env.step("RIGHT") + env.step("E") env.step("PEN_UP") for _ in range(3): - env.step("DOWN") + env.step("S") ink = ink_mask(env.canvas_pixels()) ys, xs = np.nonzero(ink) assert len(xs) > 50 @@ -63,8 +68,8 @@ def test_pen_down_moves_draw_and_pen_up_moves_do_not(env): def test_cursor_is_clamped_to_canvas(env): env.reset(seed=0) - for _ in range(30): - env.step("UP_LEFT") + for _ in range(150): + env.step("NW") assert env.cursor == (1.0, 1.0) @@ -73,7 +78,7 @@ def test_done_ends_episode_with_verifier_result(env): _, reward, done, info = env.step("DONE") assert done and reward == 0.0 and info["verifier"]["score"] == 0.0 with pytest.raises(RuntimeError): - env.step("UP") + env.step("N") def test_call_tool_draws(env): @@ -87,6 +92,7 @@ def test_call_tool_draws(env): def test_expert_passes_and_random_does_not(env): expert = play_episodes(env, circle_expert(), episodes=2, seed=0) - assert expert["pass_rate"] == 1.0 and expert["mean_score"] >= 0.7 + assert expert["pass_rate"] == 1.0 and expert["mean_score"] >= 0.85 + assert all(e["roundness"] >= 0.85 for e in expert["results"]) rnd = play_episodes(env, random_policy(0), episodes=3, seed=0) assert rnd["pass_rate"] == 0.0 and rnd["mean_score"] < expert["mean_score"] From 8b1e259201187ef189ce49bf8bb40024199c8c33 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 20:19:22 +0000 Subject: [PATCH 03/24] JSPaint env: 32-direction 6 px moves and a cleaner expert start The action set now defaults to a 6 px move toward each of 32 compass points (every 11.25 degrees, max_steps 260); 8 and 16 directions stay available. Each option's text states its bearing in degrees. The scripted expert walks straight to the nearest point on the circle and presses only within half a step of the radius, removing the nub where the stroke started off-circle. Expert score rises from 0.91 to 0.95. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- examples/jspaint_circle.py | 8 +++--- laya/games.py | 8 +++--- laya/paintenv.py | 45 +++++++++++++++++++------------ site-docs/how-to/jspaint-env.md | 47 ++++++++++++++++++++------------- tests/test_circle_verifier.py | 1 + tests/test_paintenv.py | 11 ++++---- 6 files changed, 73 insertions(+), 47 deletions(-) diff --git a/examples/jspaint_circle.py b/examples/jspaint_circle.py index 1a49deaa..e78a28cd 100644 --- a/examples/jspaint_circle.py +++ b/examples/jspaint_circle.py @@ -1,7 +1,7 @@ """Ask Laya Vision to draw a circle in JSPaint with the mouse only, and score it. Each step a screenshot of the JSPaint canvas (with the cursor marked on it) goes into the model as the image, and a -`choice` question over mouse actions (an 8 px move toward each of 16 compass directions, pen down, pen up, done) +`choice` question over mouse actions (a 6 px move toward each of 32 compass directions, pen down, pen up, done) picks the next move, which is sent to the real app as pointer events. At the end the circle verifier scores the true canvas pixels. The scripted expert and a random policy play the same seeds as reference points, and the model's score is also reported normalized, (model - random) / (expert - random), as in the games suite. @@ -31,9 +31,9 @@ def main(): ap.add_argument("--policies", default="model,expert,random", help="comma list of model, expert, random") ap.add_argument("--episodes", type=int, default=5) ap.add_argument("--seed", type=int, default=900_000) - ap.add_argument("--max-steps", type=int, default=200) - ap.add_argument("--step-px", type=int, default=8) - ap.add_argument("--directions", type=int, default=16, choices=(8, 16)) + ap.add_argument("--max-steps", type=int, default=260) + ap.add_argument("--step-px", type=int, default=6) + ap.add_argument("--directions", type=int, default=32, choices=(8, 16, 32)) ap.add_argument("--model", default="thaitea/laya-vision") ap.add_argument("--revision", default=None, help="pin the Hub revision of --model") ap.add_argument("--device", default=None) diff --git a/laya/games.py b/laya/games.py index 2b6b3dd2..909ac404 100644 --- a/laya/games.py +++ b/laya/games.py @@ -91,12 +91,14 @@ def snake_question() -> Dict: PAINT_GOALS = {"circle": "draw one round, closed circle, about as big as a third of the canvas height"} -def paint_question(task: str = "circle", directions: int = 16, step_px: int = 8) -> Dict: +def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) -> Dict: """The question for ``laya.paintenv.JSPaintEnv``: a paint canvas, a red cursor, and mouse-only actions (a ``step_px`` move toward each of ``directions`` compass points, then pen down / pen up / done).""" - from laya.paintenv import compass_moves + from laya.paintenv import compass_bearing, compass_moves - moves = {m: "move the cursor %d pixels %s" % (step_px, PAINT_DIRECTIONS[m]) for m in compass_moves(directions)} + moves = {m: "move the cursor %d pixels at bearing %g degrees (clockwise from straight up)%s" % ( + step_px, compass_bearing(m), ", " + PAINT_DIRECTIONS[m] if m in PAINT_DIRECTIONS else "") + for m in compass_moves(directions)} return {"action": { "type": "choice", "instructions": "You are using a paint program with only the mouse. The white area is the canvas and the " diff --git a/laya/paintenv.py b/laya/paintenv.py index 8d9976d1..387d5382 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -9,8 +9,8 @@ a red ring when the pen is up, a filled red dot when it is down. ``note()`` carries the pen state and step count as text; * **actions** are mouse-only. ``env.actions`` is the discrete set the image model chooses from: a ``step_px`` move - toward each of ``directions`` compass points (16 by default: ``N``, ``NNE``, ``NE``, ... every 22.5 degrees, 8 px, - fine enough to walk a round circle; 8 gives the coarse ``N NE E ...`` set), plus ``PEN_DOWN``, ``PEN_UP`` and + toward each of ``directions`` compass points (32 by default: ``N``, ``NbE``, ``NNE``, ... every 11.25 degrees, 6 px, + fine enough to walk a round circle; 16 and 8 give coarser sets), plus ``PEN_DOWN``, ``PEN_UP`` and ``DONE``. Each maps onto the tool API ``move_mouse(dx, dy)`` / ``mouse_down()`` / ``mouse_up()``, which sends real pointer events to the page. ``TOOLS`` describes that API as JSON-schema tools, so a tool-calling agent can drive the same environment with ``call_tool``; @@ -39,20 +39,28 @@ import numpy as np -COMPASS = ("N", "NNE", "NE", "ENE", "E", "ESE", "SE", "SSE", "S", "SSW", "SW", "WSW", "W", "WNW", "NW", "NNW") +COMPASS = ("N", "NbE", "NNE", "NEbN", "NE", "NEbE", "ENE", "EbN", "E", "EbS", "ESE", "SEbE", "SE", "SEbS", "SSE", + "SbE", "S", "SbW", "SSW", "SWbS", "SW", "SWbW", "WSW", "WbS", "W", "WbN", "WNW", "NWbW", "NW", "NWbN", "NNW", + "NbW") PEN_ACTIONS = ("PEN_DOWN", "PEN_UP", "DONE") -def compass_moves(directions: int = 16) -> Dict[str, Tuple[float, float]]: - """``{name: (ux, uy)}`` unit vectors (screen y points down) for 8 or 16 compass directions, clockwise from ``N``.""" - if directions not in (8, 16): - raise ValueError("directions must be 8 or 16") - every = 16 // directions - return {COMPASS[i]: (round(math.sin(math.radians(22.5 * i)), 6), round(-math.cos(math.radians(22.5 * i)), 6)) - for i in range(0, 16, every)} +def compass_bearing(name: str) -> float: + """Degrees clockwise from straight up (``N``) of a ``COMPASS`` point.""" + return COMPASS.index(name) * 11.25 -MOVES = compass_moves(16) +def compass_moves(directions: int = 32) -> Dict[str, Tuple[float, float]]: + """``{name: (ux, uy)}`` unit vectors (screen y points down) for 8, 16 or 32 compass points, clockwise from ``N`` + (the 32-point names add the "by" points, e.g. ``NbE`` is one point east of north).""" + if directions not in (8, 16, 32): + raise ValueError("directions must be 8, 16 or 32") + every = 32 // directions + return {COMPASS[i]: (round(math.sin(math.radians(11.25 * i)), 6), round(-math.cos(math.radians(11.25 * i)), 6)) + for i in range(0, 32, every)} + + +MOVES = compass_moves(32) ACTIONS = tuple(MOVES) + PEN_ACTIONS TASKS = {"circle": "Draw a circle on the canvas."} @@ -107,7 +115,7 @@ def __exit__(self, *exc): class JSPaintEnv: """One headless JSPaint page; ``reset`` starts an episode and ``step(action)`` plays one of ``env.actions``.""" - def __init__(self, url: str, task: str = "circle", step_px: int = 8, directions: int = 16, max_steps: int = 200, + def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: int = 32, max_steps: int = 260, reward: str = "terminal", headless: bool = True, executable_path: Optional[str] = None, viewport: Tuple[int, int] = (800, 600), keep_frames: bool = False): if task not in TASKS: @@ -307,7 +315,8 @@ def outcome(a): def circle_expert(radius_frac: float = 0.3): - """A scripted policy: walk (pen up) to the rightmost point of a circle centred on the canvas, press, trace one + """A scripted policy: walk (pen up) straight to the nearest point of a circle centred on the canvas, press once + within half a step of its radius, trace one full turn plus a little overlap (each step takes the forward move that stays closest to the radius), release, ``DONE``. Deterministic given the environment state.""" state: Dict = {} @@ -321,9 +330,10 @@ def policy(env: JSPaintEnv) -> str: cx, cy = state["c"] r, pos, step = state["r"], env.cursor, env.step_px if state["phase"] == "approach": - start = (cx + r, cy) - if math.dist(pos, start) > step * 0.75: - return _toward(env.moves, pos, start) + d = math.dist(pos, (cx, cy)) + if abs(d - r) > step * 0.5: + ux, uy = ((pos[0] - cx) / d, (pos[1] - cy) / d) if d > 1e-6 else (1.0, 0.0) + return _toward(env.moves, pos, (cx + r * ux, cy + r * uy)) state["phase"] = "trace" state["prev"] = math.atan2(pos[1] - cy, pos[0] - cx) return "PEN_DOWN" @@ -390,6 +400,7 @@ def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step "mean_steps": float(np.mean([e["steps"] for e in eps])), "actions": dict(counts), "results": eps} -__all__ = ["ACTIONS", "MOVES", "COMPASS", "PEN_ACTIONS", "compass_moves", "TASKS", "TOOLS", "JSPaintServer", +__all__ = ["ACTIONS", "MOVES", "COMPASS", "PEN_ACTIONS", "compass_bearing", "compass_moves", "TASKS", "TOOLS", + "JSPaintServer", "JSPaintEnv", "circle_expert", "random_policy", "ModelPolicy", "model_policy", "play_episodes", "default_chromium"] diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index 718656bc..2a8f75de 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -37,15 +37,27 @@ the cursor is placed at a seeded point in the middle of the canvas. | | | |---|---| | Observation | A screenshot of the 683×384 canvas with the cursor drawn on it: a red ring and cross when the button is up, a filled red dot when it is held down. The `note` text gives the task, the pen state and the step count. | -| Actions (`choice`) | 16 compass moves, `N` `NNE` `NE` `ENE` `E` … `NNW`, one every 22.5°. Each moves 8 px (`step_px`) and draws while the pen is down. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | -| End | The episode ends on `DONE` or after 200 steps (`max_steps`). The pen is released automatically at the end. | +| Actions (`choice`) | 32 compass moves, `N` `NbE` `NNE` `NEbN` `NE` … `NbW`, one every 11.25°. Each moves 6 px (`step_px`) and draws while the pen is down. Each option's text gives its bearing in degrees. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | +| End | The episode ends on `DONE` or after 260 steps (`max_steps`). The pen is released automatically at the end. | | Reward | The verifier score, paid at the end. With `reward="shaped"`, each step instead earns the change in score. | The model can only pick from a list of options, so the actions are relative moves. They are fine-grained because -coarse moves can't draw a circle. With 8 directions and 24 px steps, the best walk is an octagon, which scores 0.39 -and fails. With 16 directions, 12 px steps score 0.83 and 8 px steps score 0.91. Both `directions=8|16` and -`step_px` are settable (`--directions`, `--step-px`). Smaller steps draw rounder circles but take more steps. The screenshot has no OS -cursor, so the cursor is drawn onto it. This is the same approach as Maze and Snake. +coarse moves can't draw a round circle. Scripted-expert scores over 3 episodes: + +| Directions | Step | Expert score | Steps per circle | +|---|---|---|---| +| 8 | 24 px | 0.39 (an octagon; fails) | 42 | +| 16 | 12 px | 0.83 | 78 | +| 16 | 8 px | 0.91 | 116 | +| 32 | 8 px | 0.94 | 116 | +| **32** | **6 px (default)** | **0.95** | **139** | +| 32 | 4 px | 0.96 | 227 | + +More directions matter most, because the heading error per step falls from 22.5° to 5.6°. Steps below 6 px add +little and make episodes longer. Both settings are configurable (`directions=8|16|32`, `step_px`; `--directions`, +`--step-px` on the command line). + +The screenshot has no OS cursor, so the cursor is drawn onto it. This is the same approach as Maze and Snake. The discrete actions are built on a mouse-only tool API: `move_mouse(dx, dy)`, `mouse_down()`, `mouse_up()` and `screenshot()`. `paintenv.TOOLS` describes these tools as JSON-schema tool definitions, and @@ -70,22 +82,21 @@ counted as ink. It fits a least-squares circle to the ink and computes five comp `tests/test_circle_verifier.py` checks both directions: -- These pass: a drawn ring, a small ring, a coloured ring, and a 16-sided polygon (what the fine moves draw). -- These all fail: an octagon, a line, a dot, a tiny ring, a half arc, a three-quarter arc, a square, a flat ellipse, a filled - disc, a ring with a stray line, and random scribbles. +- These pass: a drawn ring, a small ring, a coloured ring, and 16- and 32-sided polygons (what the fine moves draw). +- These all fail: an octagon, a line, a dot, a tiny ring, a half arc, a three-quarter arc, a square, a flat ellipse, + a filled disc, a ring with a stray line, and random scribbles. ## Reference points -`circle_expert()` is a scripted policy. It walks with the pen up to the edge of a circle centred on the canvas, -presses the button, and traces one full turn. Each step it takes the forward move that lands closest to the radius. `random_policy` picks actions uniformly. On seeds -900000–900002: +`circle_expert()` is a scripted policy. It walks with the pen up straight to the nearest point of a circle centred +on the canvas, and presses the button once it is within half a step of the radius. It then traces one full turn, +each step taking the forward move that lands closest to the radius. `random_policy` picks actions uniformly. On +seeds 900000–900002, with the default 32 directions and 6 px steps: | Policy | Mean score | Pass rate | Mean steps | |---|---|---|---| -| Scripted expert | 0.91 | 3/3 | 116.3 | -| Random | 0.00 | 0/3 | 10.7 | -| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 167.3 | +| Scripted expert | 0.95 | 3/3 | 139.0 | +| Random | 0.00 | 0/3 | 22.0 | -The zero-shot checkpoint almost always picks `PEN_UP` (496 of 502 actions), which does nothing while the pen is -already up. It runs out the step limit without drawing anything. The model was never trained on this task. The expert's trajectories -(screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. +The model was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to +train it on, as the game checkpoints were trained on expert frames. diff --git a/tests/test_circle_verifier.py b/tests/test_circle_verifier.py index bdd18d64..faeb26de 100644 --- a/tests/test_circle_verifier.py +++ b/tests/test_circle_verifier.py @@ -34,6 +34,7 @@ def test_fit_circle_recovers_centre_and_radius(): @pytest.mark.parametrize("name,draw", [ ("ring", lambda d: d.ellipse([230, 80, 450, 300], outline="black", width=4)), ("16-gon", lambda d: d.line(polygon(16), fill="black", width=4)), + ("32-gon", lambda d: d.line(polygon(32), fill="black", width=4)), ("small ring", lambda d: d.ellipse([305, 155, 375, 225], outline="black", width=4)), ("coloured ring", lambda d: d.ellipse([100, 50, 300, 250], outline=(200, 0, 0), width=6)), ]) diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py index 07986426..8b1cde3f 100644 --- a/tests/test_paintenv.py +++ b/tests/test_paintenv.py @@ -34,10 +34,11 @@ def env(): def test_question_covers_actions_and_tools_are_mouse_only(): assert set(paint_question()["action"]["criteria"]) == set(ACTIONS) - assert len(ACTIONS) == 16 + 3 + assert len(ACTIONS) == 32 + 3 assert set(paint_question(directions=8)["action"]["criteria"]) == set(compass_moves(8)) | {"PEN_DOWN", "PEN_UP", "DONE"} - for ux, uy in compass_moves(16).values(): + assert set(compass_moves(16)) < set(compass_moves(32)) + for ux, uy in compass_moves(32).values(): assert abs(ux * ux + uy * uy - 1) < 1e-5 assert {t["name"] for t in TOOLS} == {"move_mouse", "mouse_down", "mouse_up", "screenshot"} @@ -68,7 +69,7 @@ def test_pen_down_moves_draw_and_pen_up_moves_do_not(env): def test_cursor_is_clamped_to_canvas(env): env.reset(seed=0) - for _ in range(150): + for _ in range(200): env.step("NW") assert env.cursor == (1.0, 1.0) @@ -92,7 +93,7 @@ def test_call_tool_draws(env): def test_expert_passes_and_random_does_not(env): expert = play_episodes(env, circle_expert(), episodes=2, seed=0) - assert expert["pass_rate"] == 1.0 and expert["mean_score"] >= 0.85 - assert all(e["roundness"] >= 0.85 for e in expert["results"]) + assert expert["pass_rate"] == 1.0 and expert["mean_score"] >= 0.9 + assert all(e["roundness"] >= 0.9 for e in expert["results"]) rnd = play_episodes(env, random_policy(0), episodes=3, seed=0) assert rnd["pass_rate"] == 0.0 and rnd["mean_score"] < expert["mean_score"] From aeafd6ab639b4e5cd4fd4ef9cbafbe382c580708 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 20:33:55 +0000 Subject: [PATCH 04/24] Docs: zero-shot model row for the 32-direction JSPaint defaults Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- site-docs/how-to/jspaint-env.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index 2a8f75de..d3ea9194 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -97,6 +97,8 @@ seeds 900000–900002, with the default 32 directions and 6 px steps: |---|---|---|---| | Scripted expert | 0.95 | 3/3 | 139.0 | | Random | 0.00 | 0/3 | 22.0 | +| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 260.0 | -The model was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to +The zero-shot checkpoint picks `PEN_UP` on every step (780 of 780), which does nothing while the pen is already +up. It runs out the step limit without drawing anything. The model was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. From 56e9d50ccd4b92190de76129609dfcfba38b7506 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:19:55 +0000 Subject: [PATCH 05/24] JSPaint env: square canvas the size of laya-vision's input (512x512) The released checkpoints read each image as one 512x512 view and the processor resizes to it without keeping the aspect ratio, so the 683x384 JSPaint canvas reached the model squashed: a drawn circle arrived as a tall ellipse and the model called a perfect ring "blank". The environment now sets JSPaint's canvas to canvas_size x canvas_size (default 512) through localStorage before the page loads (JSPaint is unmodified), uses a 900x720 viewport so it fits, and checks both at reset. The runner sizes the canvas from the loaded model's image_size. On the 512 canvas laya-vision recognizes the scripted expert's circles uncropped ("circle" at 0.67-0.94) and still calls a square a square; the expert scores 0.97. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- examples/jspaint_circle.py | 17 +++++++++---- laya/paintenv.py | 15 +++++++++++- site-docs/how-to/jspaint-env.md | 42 +++++++++++++++++++++++++++------ tests/test_paintenv.py | 2 +- 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/examples/jspaint_circle.py b/examples/jspaint_circle.py index e78a28cd..ac6e2050 100644 --- a/examples/jspaint_circle.py +++ b/examples/jspaint_circle.py @@ -38,6 +38,8 @@ def main(): ap.add_argument("--revision", default=None, help="pin the Hub revision of --model") ap.add_argument("--device", default=None) ap.add_argument("--chromium", default=None, help="Chromium executable (default: LAYA_CHROMIUM or Playwright's)") + ap.add_argument("--canvas-size", type=int, default=None, + help="square canvas side (default: the model's input image_size, 512 for the released checkpoints)") ap.add_argument("--headed", action="store_true", help="show the browser window") ap.add_argument("--out", default="results/jspaint") args = ap.parse_args() @@ -47,15 +49,22 @@ def main(): names = [p.strip() for p in args.policies.split(",") if p.strip()] summary = {"task": "circle", "jspaint": os.path.abspath(args.jspaint), "args": vars(args), "policies": {}} + agent = None + if "model" in names: + from laya import load_vlm + + agent = load_vlm(args.model, revision=args.revision, device=args.device) + # A square canvas the size of the model's input, so the screenshot reaches the vision tower unscaled. + canvas_size = args.canvas_size or (agent.prep.image_size if agent is not None else 512) + summary["canvas_size"] = canvas_size + with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, max_steps=args.max_steps, step_px=args.step_px, directions=args.directions, - headless=not args.headed, + canvas_size=canvas_size, headless=not args.headed, executable_path=args.chromium, keep_frames=True) as env: for name in names: if name == "model": - from laya import load_vlm - - policy = model_policy(load_vlm(args.model, revision=args.revision, device=args.device)) + policy = model_policy(agent) elif name == "expert": policy = circle_expert() elif name == "random": diff --git a/laya/paintenv.py b/laya/paintenv.py index 387d5382..b458b2bd 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -3,6 +3,10 @@ `JSPaint `_ (an MS Paint clone) runs unmodified in headless Chromium through Playwright; ``JSPaintServer`` serves a local checkout as static files. Each episode: +* **canvas** is ``canvas_size`` pixels, square and by default 512 x 512: the size laya-vision's released checkpoints + feed the vision tower (``image_size`` in ``vlm_agent_config.json``). The model then sees the canvas 1:1; a wide + canvas would be squashed to a square first, turning a drawn circle into a tall ellipse. JSPaint reads its canvas + size from ``localStorage`` at start-up, so the size is set there before the page loads (JSPaint is unmodified); * **reset** clears the canvas to white (``api_for_cypress_tests.reset_for_next_test()``), selects the Brush tool (so strokes are 4 px wide and visible) and puts the cursor at a seeded point on the canvas; * **observation** is a screenshot of the canvas with the cursor drawn on it (headless screenshots have no OS cursor): @@ -117,7 +121,7 @@ class JSPaintEnv: def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: int = 32, max_steps: int = 260, reward: str = "terminal", headless: bool = True, executable_path: Optional[str] = None, - viewport: Tuple[int, int] = (800, 600), keep_frames: bool = False): + viewport: Tuple[int, int] = (900, 720), keep_frames: bool = False, canvas_size: int = 512): if task not in TASKS: raise ValueError("unknown task %r (one of %s)" % (task, ", ".join(TASKS))) if reward not in ("terminal", "shaped"): @@ -132,6 +136,9 @@ def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: self.browser = self._pw.chromium.launch(headless=headless, executable_path=executable_path or default_chromium()) self.page = self.browser.new_page(viewport={"width": viewport[0], "height": viewport[1]}) + self.canvas_size = canvas_size + self.page.add_init_script("localStorage.setItem('width', '%d'); localStorage.setItem('height', '%d');" + % (canvas_size, canvas_size)) # Only the app itself: no fonts, analytics or update checks from the network. self.page.route("**/*", lambda route: route.continue_() if route.request.url.startswith(url) else route.abort()) @@ -154,6 +161,12 @@ def reset(self, seed: int = 0): r = self.page.evaluate("(() => { const r = document.querySelector('.main-canvas').getBoundingClientRect();" " return [r.x, r.y, r.width, r.height]; })()") self.origin, self.width, self.height = (r[0], r[1]), int(r[2]), int(r[3]) + if (self.width, self.height) != (self.canvas_size, self.canvas_size): + raise RuntimeError("JSPaint canvas is %dx%d, expected %dx%d" % (self.width, self.height, self.canvas_size, + self.canvas_size)) + if r[1] + r[3] > self.page.viewport_size["height"] or r[0] + r[2] > self.page.viewport_size["width"]: + raise RuntimeError("the %dx%d canvas does not fit the browser viewport; pass a larger viewport" + % (self.width, self.height)) rng = random.Random(seed) self.cursor = (rng.uniform(0.25, 0.75) * self.width, rng.uniform(0.25, 0.75) * self.height) self.page.mouse.move(*self._client(self.cursor)) diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index d3ea9194..e72f0934 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -36,13 +36,43 @@ the cursor is placed at a seeded point in the middle of the canvas. | | | |---|---| -| Observation | A screenshot of the 683×384 canvas with the cursor drawn on it: a red ring and cross when the button is up, a filled red dot when it is held down. The `note` text gives the task, the pen state and the step count. | +| Canvas | 512×512, the size of laya-vision's input image (`--canvas-size`, default: the model's `image_size`). | +| Observation | A screenshot of the canvas with the cursor drawn on it: a red ring and cross when the button is up, a filled red dot when it is held down. The `note` text gives the task, the pen state and the step count. | | Actions (`choice`) | 32 compass moves, `N` `NbE` `NNE` `NEbN` `NE` … `NbW`, one every 11.25°. Each moves 6 px (`step_px`) and draws while the pen is down. Each option's text gives its bearing in degrees. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | | End | The episode ends on `DONE` or after 260 steps (`max_steps`). The pen is released automatically at the end. | | Reward | The verifier score, paid at the end. With `reward="shaped"`, each step instead earns the change in score. | +### Why the canvas is 512×512 + +The released checkpoints read every image as a single 512×512 view, and the processor resizes to that square +without keeping the aspect ratio. JSPaint's default 683×384 canvas was therefore squashed: a drawn circle reached the +model as a tall ellipse (165×293 px). On that canvas laya-vision called a perfect ring "blank", with P(circle) 0.05. + +A 512×512 canvas reaches the vision tower pixel for pixel. On it, the model recognizes the scripted expert's circles +without any cropping, and still tells them apart from a square drawn the same way: + +| Canvas (JSPaint, 512×512) | "What is drawn?" (6 options) | "Which shape?" (5 options) | Verifier | +|---|---|---|---| +| Expert circle, seed 0 | circle (0.67) | circle (0.94) | 0.97 | +| Expert circle, seed 1 | circle (0.84) | circle (0.94) | 0.97 | +| Expert circle, seed 2 | circle (0.82) | circle (0.94) | 0.97 | +| Square drawn with the same moves | square (P(circle) 0.09) | square (0.03) | 0.00 | +| Blank | blank (0.08) | heart (0.15) | 0.00 | + +Each cell is the top answer, with the probability of "circle" in brackets. These are spot checks of +`thaitea/laya-vision` at revision `f2fe3c1`, not a benchmark. Two caveats. The yes/no question "Does the image +show a circle?" stays below 0.35 on all of them, so ask it as a choice. And the red cursor marker in the +observation sometimes pulls the 6-option answer to "line". + +JSPaint reads its canvas size from `localStorage` at start-up, so the environment writes `width` and `height` +there before the page loads. JSPaint itself is unmodified. The browser viewport is 900×720 so the whole canvas is +on screen. + +### Move size + The model can only pick from a list of options, so the actions are relative moves. They are fine-grained because -coarse moves can't draw a round circle. Scripted-expert scores over 3 episodes: +coarse moves can't draw a round circle. Scripted-expert scores over 3 episodes, measured on the earlier 683×384 +canvas (radius 115 px): | Directions | Step | Expert score | Steps per circle | |---|---|---|---| @@ -91,14 +121,12 @@ counted as ink. It fits a least-squares circle to the ink and computes five comp `circle_expert()` is a scripted policy. It walks with the pen up straight to the nearest point of a circle centred on the canvas, and presses the button once it is within half a step of the radius. It then traces one full turn, each step taking the forward move that lands closest to the radius. `random_policy` picks actions uniformly. On -seeds 900000–900002, with the default 32 directions and 6 px steps: +seeds 900000–900002, on the 512×512 canvas with the default 32 directions and 6 px steps: | Policy | Mean score | Pass rate | Mean steps | |---|---|---|---| -| Scripted expert | 0.95 | 3/3 | 139.0 | +| Scripted expert | 0.97 | 3/3 | 181.7 | | Random | 0.00 | 0/3 | 22.0 | -| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 260.0 | -The zero-shot checkpoint picks `PEN_UP` on every step (780 of 780), which does nothing while the pen is already -up. It runs out the step limit without drawing anything. The model was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to +The model was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py index 8b1cde3f..4a3b36a3 100644 --- a/tests/test_paintenv.py +++ b/tests/test_paintenv.py @@ -45,7 +45,7 @@ def test_question_covers_actions_and_tools_are_mouse_only(): def test_reset_is_blank_and_observation_is_canvas(env): obs = env.reset(seed=3) - assert obs.size == (env.width, env.height) + assert obs.size == (env.width, env.height) == (512, 512) assert not ink_mask(env.canvas_pixels()).any() assert not env.done and not env.pen From 1d370954ac91f715b62147228d2f9fdb33508fc9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 21:29:33 +0000 Subject: [PATCH 06/24] Docs: zero-shot model row on the 512x512 JSPaint canvas Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- site-docs/how-to/jspaint-env.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index e72f0934..9184d5ba 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -127,6 +127,8 @@ seeds 900000–900002, on the 512×512 canvas with the default 32 directions and |---|---|---|---| | Scripted expert | 0.97 | 3/3 | 181.7 | | Random | 0.00 | 0/3 | 22.0 | +| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 174.0 | -The model was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to +The zero-shot checkpoint picks `PEN_UP` on 521 of 522 steps, which does nothing while the pen is already up, and +never draws. Recognising a circle is not the same as knowing how to draw one: it was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. From b31dea78ec46e23e2a853fda2e55bc618f56fc8f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 23:28:19 +0000 Subject: [PATCH 07/24] JSPaint env: two-frame memory, recent actions, and the full question uncut Each step the model now sees the canvas frame_gap (8) steps ago and now, plus its pen state, step count and last 12 actions as state text (JSPaintEnv.state). The 35-option drawing question with full descriptions is ~1,050 tokens; the checkpoint's head_max_len of 256 had silently cut every option and 48 of the 82 instruction tokens. The runner now loads the model with PAINT_HEAD_MAX_LEN=1536 / PAINT_MAX_LEN=2560, ModelPolicy calls predict with strict=True so nothing can be cut silently, and a test checks the budget. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- examples/jspaint_circle.py | 19 +++++++++------ laya/games.py | 8 +++---- laya/paintenv.py | 41 ++++++++++++++++++++++++++++----- site-docs/how-to/jspaint-env.md | 20 ++++++++++++---- tests/test_paintenv.py | 32 +++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 21 deletions(-) diff --git a/examples/jspaint_circle.py b/examples/jspaint_circle.py index ac6e2050..e13550cd 100644 --- a/examples/jspaint_circle.py +++ b/examples/jspaint_circle.py @@ -1,10 +1,12 @@ """Ask Laya Vision to draw a circle in JSPaint with the mouse only, and score it. -Each step a screenshot of the JSPaint canvas (with the cursor marked on it) goes into the model as the image, and a -`choice` question over mouse actions (a 6 px move toward each of 32 compass directions, pen down, pen up, done) -picks the next move, which is sent to the real app as pointer events. At the end the circle verifier scores the -true canvas pixels. The scripted expert and a random policy play the same seeds as reference points, and the -model's score is also reported normalized, (model - random) / (expert - random), as in the games suite. +Each step the model sees two screenshots of the JSPaint canvas (8 steps ago and now, the cursor marked on both) plus +its pen state and last 12 actions as text, and a `choice` question over mouse actions (a 6 px move toward each of 32 +compass directions, pen down, pen up, done) picks the next move, which is sent to the real app as pointer events. +The model is loaded with raised token budgets and asked with strict=True, so the full question is never cut. At the +end the circle verifier scores the true canvas pixels. The scripted expert and a random policy play the same seeds as +reference points, and the model's score is also reported normalized, (model - random) / (expert - random), as in the +games suite. pip install -e . playwright pillow git clone https://github.com/r33drichards/jspaint ../jspaint @@ -21,7 +23,8 @@ import os import time -from laya.paintenv import JSPaintEnv, JSPaintServer, circle_expert, model_policy, play_episodes, random_policy +from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, model_policy, + play_episodes, random_policy) def main(): @@ -53,7 +56,9 @@ def main(): if "model" in names: from laya import load_vlm - agent = load_vlm(args.model, revision=args.revision, device=args.device) + # raised token budgets: the full question and a two-frame state fit with nothing cut + agent = load_vlm(args.model, revision=args.revision, device=args.device, head_max_len=PAINT_HEAD_MAX_LEN, + max_len=PAINT_MAX_LEN) # A square canvas the size of the model's input, so the screenshot reaches the vision tower unscaled. canvas_size = args.canvas_size or (agent.prep.image_size if agent is not None else 512) summary["canvas_size"] = canvas_size diff --git a/laya/games.py b/laya/games.py index 909ac404..f58cd0ee 100644 --- a/laya/games.py +++ b/laya/games.py @@ -101,10 +101,10 @@ def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) for m in compass_moves(directions)} return {"action": { "type": "choice", - "instructions": "You are using a paint program with only the mouse. The white area is the canvas and the " - "red mark is the mouse cursor: a hollow ring with a cross means the button is up, a filled " - "dot means it is held down and moving draws a black line. Your task: %s. Which mouse action " - "should you take now?" % PAINT_GOALS[task], + "instructions": "You are using a paint program with only the mouse. You see the canvas a few steps ago " + "and now. The white area is the canvas and the red mark is the mouse cursor: a hollow ring " + "with a cross means the button is up, a filled dot means it is held down and moving draws a " + "black line. Your task: %s. Which mouse action should you take now?" % PAINT_GOALS[task], "criteria": {**moves, **PAINT_PEN}, }} diff --git a/laya/paintenv.py b/laya/paintenv.py index b458b2bd..c3e4031f 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -18,6 +18,9 @@ ``DONE``. Each maps onto the tool API ``move_mouse(dx, dy)`` / ``mouse_down()`` / ``mouse_up()``, which sends real pointer events to the page. ``TOOLS`` describes that API as JSON-schema tools, so a tool-calling agent can drive the same environment with ``call_tool``; +* **memory**: ``state()`` is what the model sees each step: two images, the canvas ``frame_gap`` steps ago and + now (so the direction of travel is visible), plus the pen state, step count and the last ``history`` actions as + text. One screenshot alone does not say which way the stroke was going or what was just tried; * **reward** is the task verifier on the true canvas pixels (``canvas_pixels()``, read from the page, so the cursor overlay never counts as ink): ``laya.circle_verifier.score_circle`` for ``task="circle"``. It is paid at the end of the episode, or as the per-step change in score with ``reward="shaped"``. An episode ends on ``DONE`` or after @@ -38,7 +41,7 @@ import os import random import threading -from collections import Counter +from collections import Counter, deque from typing import Dict, List, Optional, Tuple import numpy as np @@ -82,6 +85,10 @@ def compass_moves(directions: int = 32) -> Dict[str, Tuple[float, float]]: ] CURSOR = (230, 30, 30) +# Token budgets for ``load_vlm`` so the full drawing question (35 options with their descriptions, ~1050 tokens) and +# a two-image state fit without any cutting; the checkpoint defaults (256 / 1024) would truncate every option. +PAINT_HEAD_MAX_LEN = 1536 +PAINT_MAX_LEN = 2560 def default_chromium() -> Optional[str]: @@ -121,7 +128,8 @@ class JSPaintEnv: def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: int = 32, max_steps: int = 260, reward: str = "terminal", headless: bool = True, executable_path: Optional[str] = None, - viewport: Tuple[int, int] = (900, 720), keep_frames: bool = False, canvas_size: int = 512): + viewport: Tuple[int, int] = (900, 720), keep_frames: bool = False, canvas_size: int = 512, + frame_gap: int = 8, history: int = 12): if task not in TASKS: raise ValueError("unknown task %r (one of %s)" % (task, ", ".join(TASKS))) if reward not in ("terminal", "shaped"): @@ -130,6 +138,8 @@ def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: self.url, self.task, self.step_px, self.max_steps, self.reward_mode = url, task, step_px, max_steps, reward self.keep_frames, self.directions = keep_frames, directions + self.frame_gap, self.history = frame_gap, history + self._recent_obs: deque = deque(maxlen=frame_gap + 1) self.moves = compass_moves(directions) self.actions = tuple(self.moves) + PEN_ACTIONS self._pw = sync_playwright().start() @@ -174,6 +184,8 @@ def reset(self, seed: int = 0): self.seed, self.steps, self.pen, self.finished = seed, 0, False, False self.trajectory, self.frames, self._last_score, self.result = [], [], 0.0, None obs = self.render() + self._recent_obs.clear() + self._recent_obs.append(obs) if self.keep_frames: self.frames.append(obs) return obs @@ -212,6 +224,7 @@ def step(self, action: str): score = self.verify()["score"] reward, self._last_score = score - self._last_score, score obs = self.render() + self._recent_obs.append(obs) if self.keep_frames: self.frames.append(obs) return obs, reward, self.done, info @@ -282,6 +295,18 @@ def render(self, full_page: bool = False): d.line([x, y - 11, x, y + 11], fill=CURSOR) return img + def state(self) -> Dict: + """What the model sees this step: ``images`` = [canvas ``frame_gap`` steps ago (the first frame early in an + episode), canvas now], both with the cursor drawn on, plus the pen state, step count and recent actions.""" + recent = [t["action"] for t in self.trajectory[-self.history:]] + return { + "images": [self._recent_obs[0], self._recent_obs[-1]], + "frames": "the first image is the canvas %d steps ago, the second is now" % (len(self._recent_obs) - 1), + "pen": "down (drawing)" if self.pen else "up (not drawing)", + "step": "%d of %d" % (self.steps + 1, self.max_steps), + "recent_actions": " ".join(recent) if recent else "none yet", + } + def note(self) -> str: return "Task: %s The pen is %s. Step %d of %d." % (TASKS[self.task], "down (drawing)" if self.pen else "up (not drawing)", self.steps + 1, self.max_steps) @@ -370,8 +395,11 @@ def random_policy(seed: int = 0): class ModelPolicy: - """The model's most likely action from the screenshot plus the text note, via ``predict``. ``last`` keeps the - latest answer (probabilities over ``env.actions``) for logging.""" + """The model's most likely action from ``env.state()`` (two frames plus pen state and recent actions), via + ``predict``. ``last`` keeps the latest answer (probabilities over ``env.actions``) for logging. + + Load the agent with ``head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN``: ``predict`` runs with + ``strict=True`` and raises rather than silently cut the question.""" def __init__(self, agent, task: str = "circle"): self.agent, self.task, self.last, self.provenance, self._questions = agent, task, None, None, {} @@ -382,7 +410,7 @@ def __call__(self, env: JSPaintEnv) -> str: key = (env.directions, env.step_px) if key not in self._questions: self._questions[key] = paint_question(self.task, env.directions, env.step_px) - out = self.agent.predict({"image": env.render(), "note": env.note()}, self._questions[key]) + out = self.agent.predict(env.state(), self._questions[key], strict=True) self.last, self.provenance = out["answers"]["action"], out.get("provenance") return self.last["choice"] @@ -413,7 +441,8 @@ def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step "mean_steps": float(np.mean([e["steps"] for e in eps])), "actions": dict(counts), "results": eps} -__all__ = ["ACTIONS", "MOVES", "COMPASS", "PEN_ACTIONS", "compass_bearing", "compass_moves", "TASKS", "TOOLS", +__all__ = ["PAINT_HEAD_MAX_LEN", "PAINT_MAX_LEN", "ACTIONS", "MOVES", "COMPASS", "PEN_ACTIONS", "compass_bearing", + "compass_moves", "TASKS", "TOOLS", "JSPaintServer", "JSPaintEnv", "circle_expert", "random_policy", "ModelPolicy", "model_policy", "play_episodes", "default_chromium"] diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index 9184d5ba..4de28634 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -37,7 +37,7 @@ the cursor is placed at a seeded point in the middle of the canvas. | | | |---|---| | Canvas | 512×512, the size of laya-vision's input image (`--canvas-size`, default: the model's `image_size`). | -| Observation | A screenshot of the canvas with the cursor drawn on it: a red ring and cross when the button is up, a filled red dot when it is held down. The `note` text gives the task, the pen state and the step count. | +| Observation | Two screenshots of the canvas, 8 steps ago (`frame_gap`) and now, with the cursor drawn on both: a red ring and cross when the button is up, a filled red dot when it is held down. The state text gives the pen state, the step count and the last 12 actions (`history`). `JSPaintEnv.state()` builds it. | | Actions (`choice`) | 32 compass moves, `N` `NbE` `NNE` `NEbN` `NE` … `NbW`, one every 11.25°. Each moves 6 px (`step_px`) and draws while the pen is down. Each option's text gives its bearing in degrees. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | | End | The episode ends on `DONE` or after 260 steps (`max_steps`). The pen is released automatically at the end. | | Reward | The verifier score, paid at the end. With `reward="shaped"`, each step instead earns the change in score. | @@ -87,6 +87,17 @@ More directions matter most, because the heading error per step falls from 22.5 little and make episodes longer. Both settings are configurable (`directions=8|16|32`, `step_px`; `--directions`, `--step-px` on the command line). +### Memory and the full question + +One screenshot does not show which way the stroke was heading or what the model just tried, so each step carries +an earlier frame and the recent actions as well. + +The drawing question is sent in full. It has 35 options, each with its own description, and comes to about 1,050 +tokens. The checkpoint's defaults (`head_max_len` 256, `max_len` 1024) would cut every option and half the +instructions, so the model is loaded with `head_max_len=PAINT_HEAD_MAX_LEN` (1,536) and `max_len=PAINT_MAX_LEN` +(2,560). `ModelPolicy` calls `predict(..., strict=True)`, which raises instead of cutting. The model runs before +this change were affected: every option and 48 of the 82 instruction tokens had been cut. + The screenshot has no OS cursor, so the cursor is drawn onto it. This is the same approach as Maze and Snake. The discrete actions are built on a mouse-only tool API: `move_mouse(dx, dy)`, `mouse_down()`, `mouse_up()` and @@ -127,8 +138,9 @@ seeds 900000–900002, on the 512×512 canvas with the default 32 directions and |---|---|---|---| | Scripted expert | 0.97 | 3/3 | 181.7 | | Random | 0.00 | 0/3 | 22.0 | -| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 174.0 | +| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 31.0 | -The zero-shot checkpoint picks `PEN_UP` on 521 of 522 steps, which does nothing while the pen is already up, and -never draws. Recognising a circle is not the same as knowing how to draw one: it was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to +With the full question and two-frame memory, the zero-shot checkpoint moves the cursor (mostly `NNW`, 81 of 93 +actions) but never presses the button, and ends each episode with `DONE` within 6–48 steps. Its top option gets +about 0.05 probability against 0.03 for uniform, so it is close to guessing. Recognising a circle is not the same as knowing how to draw one: it was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py index 4a3b36a3..270c8d49 100644 --- a/tests/test_paintenv.py +++ b/tests/test_paintenv.py @@ -43,6 +43,38 @@ def test_question_covers_actions_and_tools_are_mouse_only(): assert {t["name"] for t in TOOLS} == {"move_mouse", "mouse_down", "mouse_up", "screenshot"} +def test_state_has_two_frames_and_recent_actions(env): + env.reset(seed=2) + first = env.state() + assert len(first["images"]) == 2 and first["images"][0] is first["images"][1] + assert first["recent_actions"] == "none yet" + for a in ["PEN_DOWN"] + ["E"] * (env.frame_gap + 3): + env.step(a) + st = env.state() + old, now = st["images"] + assert old is not now and old.size == now.size == (env.width, env.height) + assert st["recent_actions"].split() == (["PEN_DOWN"] + ["E"] * (env.frame_gap + 3))[-env.history:] + assert st["pen"].startswith("down") + + +def test_full_question_fits_raised_budget_without_truncation(): + """The full 35-option question plus a two-frame state must fit PAINT_HEAD_MAX_LEN / PAINT_MAX_LEN uncut.""" + from transformers import AutoProcessor + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN + + try: + tok = AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM-256M-Instruct").tokenizer + except Exception as err: # offline + pytest.skip("tokenizer unavailable: %s" % err) + q = paint_question()["action"] + opts = ["- %s: %s" % kv for kv in q["criteria"].items()] + n_opts = sum(len(tok(o)["input_ids"]) + 1 for o in opts) + n_ins = len(tok(q["instructions"])["input_ids"]) + assert max(len(tok(o)["input_ids"]) for o in opts) <= 48 # the builder's per-option cap + assert n_opts + n_ins + 32 <= PAINT_HEAD_MAX_LEN + assert n_opts + n_ins + 2 * 64 + 256 <= PAINT_MAX_LEN # two 512px images are 64 tokens each + + def test_reset_is_blank_and_observation_is_canvas(env): obs = env.reset(seed=3) assert obs.size == (env.width, env.height) == (512, 512) From 0bad323b1e03abac67f5fa501af47a754c609c74 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 23:37:02 +0000 Subject: [PATCH 08/24] JSPaint env: judgement questions asked with each action paint_judgements adds progress (score 0-4), on_track (named choice) and drawn (choice over shapes, incl. oval and arc), asked about the same state in the same predict call as the action. ModelPolicy logs them with the act gate, can end the episode on its own completion judgement (judge_stop), and the runner labels each episode's clean final canvas (judged_final) for hindsight relabelling. examples/jspaint_judge_check.py compares the judgements with true progress along expert and random play. Zero-shot, drawn works (3/3 circles) but progress and on_track are nearly flat (Spearman 0.40 over a narrow range; random play rated slightly higher), so they need training; the environment can supply their labels. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- examples/jspaint_circle.py | 23 +++++-- examples/jspaint_judge_check.py | 115 ++++++++++++++++++++++++++++++++ laya/games.py | 39 +++++++++++ laya/paintenv.py | 67 +++++++++++++++---- site-docs/how-to/jspaint-env.md | 41 ++++++++++++ tests/test_paintenv.py | 16 +++++ 6 files changed, 285 insertions(+), 16 deletions(-) create mode 100644 examples/jspaint_judge_check.py diff --git a/examples/jspaint_circle.py b/examples/jspaint_circle.py index e13550cd..fb90697e 100644 --- a/examples/jspaint_circle.py +++ b/examples/jspaint_circle.py @@ -23,8 +23,10 @@ import os import time -from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, model_policy, - play_episodes, random_policy) +from PIL import Image + +from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, judge_canvas, + model_policy, play_episodes, random_policy) def main(): @@ -43,6 +45,10 @@ def main(): ap.add_argument("--chromium", default=None, help="Chromium executable (default: LAYA_CHROMIUM or Playwright's)") ap.add_argument("--canvas-size", type=int, default=None, help="square canvas side (default: the model's input image_size, 512 for the released checkpoints)") + ap.add_argument("--no-judge", action="store_true", + help="skip the judgement questions (progress, on track, what is drawn) the model answers each step") + ap.add_argument("--judge-stop", action="store_true", + help="let the model's own judgement end the episode: DONE once it rates the task complete") ap.add_argument("--headed", action="store_true", help="show the browser window") ap.add_argument("--out", default="results/jspaint") args = ap.parse_args() @@ -69,7 +75,7 @@ def main(): executable_path=args.chromium, keep_frames=True) as env: for name in names: if name == "model": - policy = model_policy(agent) + policy = model_policy(agent, judge=not args.no_judge, judge_stop=args.judge_stop) elif name == "expert": policy = circle_expert() elif name == "random": @@ -77,14 +83,20 @@ def main(): else: raise SystemExit("unknown policy %r" % name) - rows = [] + rows, final_labels = [], {} def on_step(env, action): row = dict(env.trajectory[-1], episode=env.seed) if name == "model": row["probabilities"] = policy.last["probabilities"] + row["act_probability"] = policy.last["action"]["act_probability"] + if policy.last_judgement: + row["judgement"] = policy.last_judgement rows.append(row) if env.done: + if agent is not None and not args.no_judge: + # what the clean final canvas (no cursor) actually shows, for hindsight relabelling + final_labels[env.seed] = judge_canvas(agent, Image.fromarray(env.canvas_pixels())) ep_dir = os.path.join(out_dir, name, "seed%d" % env.seed) os.makedirs(ep_dir) with open(os.path.join(ep_dir, "trajectory.jsonl"), "w") as f: @@ -97,6 +109,9 @@ def on_step(env, action): t = time.time() res = play_episodes(env, policy, args.episodes, seed=args.seed, on_step=on_step) res["seconds"] = round(time.time() - t, 1) + for ep in res["results"]: + if ep["seed"] in final_labels: + ep["judged_final"] = final_labels[ep["seed"]] if name == "model": res["model"], res["provenance"] = args.model, policy.provenance summary["policies"][name] = res diff --git a/examples/jspaint_judge_check.py b/examples/jspaint_judge_check.py new file mode 100644 index 00000000..40e138dc --- /dev/null +++ b/examples/jspaint_judge_check.py @@ -0,0 +1,115 @@ +"""Check whether Laya Vision's judgements track what is really happening on the JSPaint canvas. + +The scripted circle expert and a random policy play; every --every steps the model answers the judgement questions +(`laya.games.paint_judgements`: progress 0-4, on track or off track, what is drawn) about the same two-frame state +the drawing policy sees. The truth for progress is the share of the 36 angular sectors around the target circle's +centre that the cursor has passed through with the pen down (0 before drawing, 1 for a full turn). Reported: + +* how judged progress rises with true progress along the expert's circles (Spearman rank correlation, and the mean + judged progress per quarter of true progress); +* judged progress and P(on track) for the random policy, which should stay low; +* what the model says the clean final canvas shows, per policy. + + python examples/jspaint_judge_check.py --jspaint ../jspaint --episodes 3 + +Writes a new directory under --out: judgements.jsonl (one row per check) and summary.json. +""" +import argparse +import json +import math +import os +import time + +import numpy as np +from PIL import Image + +from laya.games import paint_judgements +from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, judge_canvas, + random_policy, summarize_judgements) + +SECTORS = 36 + + +def true_progress(env) -> float: + """Share of the 36 sectors around the canvas centre that pen-down cursor positions have covered.""" + cx, cy = env.width / 2.0, env.height / 2.0 + hit = {int((math.atan2(t["y"] - cy, t["x"] - cx) + math.pi) / (2 * math.pi) * SECTORS) % SECTORS + for t in env.trajectory if t["pen"]} + return len(hit) / SECTORS + + +def spearman(a, b) -> float: + ra, rb = np.argsort(np.argsort(a)), np.argsort(np.argsort(b)) + return float(np.corrcoef(ra, rb)[0, 1]) if len(a) > 2 and np.std(ra) and np.std(rb) else float("nan") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--jspaint", default=os.path.join(os.path.dirname(__file__), "..", "..", "jspaint")) + ap.add_argument("--episodes", type=int, default=3) + ap.add_argument("--every", type=int, default=10, help="ask the judgements every N steps") + ap.add_argument("--seed", type=int, default=900_000) + ap.add_argument("--model", default="thaitea/laya-vision") + ap.add_argument("--revision", default=None) + ap.add_argument("--device", default=None) + ap.add_argument("--chromium", default=None) + ap.add_argument("--out", default="results/jspaint-judge") + args = ap.parse_args() + + from laya import load_vlm + + out_dir = os.path.join(args.out, time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(out_dir, exist_ok=False) + agent = load_vlm(args.model, revision=args.revision, device=args.device, head_max_len=PAINT_HEAD_MAX_LEN, + max_len=PAINT_MAX_LEN) + questions = paint_judgements("circle") + rows, finals = [], {} + with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size, + executable_path=args.chromium) as env: + for name, make in (("expert", lambda: circle_expert()), ("random", lambda: random_policy(args.seed))): + policy = make() + finals[name] = [] + for i in range(args.episodes): + env.reset(args.seed + i) + while not env.done: + if env.steps % args.every == 0: + ans = agent.predict(env.state(), questions, strict=True)["answers"] + rows.append({"policy": name, "seed": args.seed + i, "step": env.steps, "pen": env.pen, + "true_progress": round(true_progress(env), 4), **summarize_judgements(ans)}) + env.step(policy(env)) + final = judge_canvas(agent, Image.fromarray(env.canvas_pixels())) + finals[name].append({"seed": args.seed + i, "verifier": env.result["score"], **final}) + rows.append({"policy": name, "seed": args.seed + i, "step": env.steps, "final": True, + "true_progress": round(true_progress(env), 4), **final}) + print("%-6s seed %d verifier %.2f final canvas judged: %s (progress %.2f, on track %.2f)" % ( + name, args.seed + i, env.result["score"], final["drawn"], final["progress"], final["on_track"])) + + exp = [r for r in rows if r["policy"] == "expert" and not r.get("final")] + rnd = [r for r in rows if r["policy"] == "random"] + tp, jp = [r["true_progress"] for r in exp], [r["progress"] for r in exp] + quarters = {} + for lo, hi in ((0, 0.001), (0.001, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.01)): + sel = [r for r in exp if lo <= r["true_progress"] < hi] + if sel: + quarters["%.2f-%.2f" % (lo, min(hi, 1.0))] = { + "n": len(sel), "judged_progress": round(float(np.mean([r["progress"] for r in sel])), 3), + "p_on_track": round(float(np.mean([r["on_track"] for r in sel])), 3)} + summary = { + "model": args.model, "provenance_revision": getattr(agent, "source", {}).get("revision"), + "every": args.every, "episodes": args.episodes, "seed": args.seed, + "expert_spearman_true_vs_judged_progress": round(spearman(tp, jp), 3), + "expert_by_true_progress": quarters, + "random_mean_judged_progress": round(float(np.mean([r["progress"] for r in rnd])), 3), + "random_mean_p_on_track": round(float(np.mean([r["on_track"] for r in rnd])), 3), + "final_canvas": finals, + } + with open(os.path.join(out_dir, "judgements.jsonl"), "w") as f: + f.writelines(json.dumps(r) + "\n" for r in rows) + with open(os.path.join(out_dir, "summary.json"), "w") as f: + json.dump(summary, f, indent=2) + print(json.dumps({k: v for k, v in summary.items() if k != "final_canvas"}, indent=2)) + print("wrote %s" % out_dir) + + +if __name__ == "__main__": + main() diff --git a/laya/games.py b/laya/games.py index f58cd0ee..b7ca27a0 100644 --- a/laya/games.py +++ b/laya/games.py @@ -109,6 +109,45 @@ def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) }} +PAINT_SHAPES = { + "circle": "a round, closed circle", "oval": "an oval or ellipse, round but stretched", + "arc": "a curved line or an unclosed part of a circle", "line": "one or more straight lines", + "square": "a square or rectangle", "triangle": "a triangle", "scribble": "random scribbles", + "nothing": "nothing, the canvas is blank", +} + + +def paint_judgements(task: str = "circle") -> Dict: + """The judgement questions asked with ``paint_question`` each step (same state, same ``predict`` call): + + * ``progress`` (``score``, 0-4): how far the canvas is toward the task, the model's own dense signal; + * ``on_track`` (``choice``): whether the drawing so far is heading toward the task. Named options, because on + line drawings the model's yes/no (``noul``) answers were unreliable while named choices were not; + * ``drawn`` (``choice`` over ``PAINT_SHAPES``): what the canvas shows now, whatever the task. At the end of an + episode this labels what was actually drawn, so a failed attempt can be relabelled as an example of that. + """ + goal = PAINT_GOALS[task] + return { + "progress": { + "type": "score", + "instructions": "The task is to %s. Looking at the canvas now, how far along is the drawing?" % goal, + "criteria": ["nothing useful drawn yet", "started, but far from done", "about half done", + "nearly done", "the task is complete"], + }, + "on_track": { + "type": "choice", + "instructions": "The task is to %s. Is the drawing so far heading toward that?" % goal, + "criteria": {"on track": "what is drawn so far could become the task by continuing", + "off track": "what is drawn so far will not become the task"}, + }, + "drawn": { + "type": "choice", + "instructions": "What is drawn on this white canvas?", + "criteria": dict(PAINT_SHAPES), + }, + } + + CONTROL_GOALS = { "CartPole": "A pole is hinged on a cart; push the cart left or right to keep the pole upright and the cart " "on screen.", diff --git a/laya/paintenv.py b/laya/paintenv.py index c3e4031f..67a06179 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -394,29 +394,72 @@ def random_policy(seed: int = 0): return lambda env: rng.choice(env.actions) +def summarize_judgements(answers: Dict) -> Dict: + """The judgement answers from ``predict`` as a compact record: expected progress (0-4) and its distribution, + P(on track), and the top ``drawn`` label with its distribution.""" + out = {} + if "progress" in answers: + out["progress"] = answers["progress"]["score"] + out["progress_probs"] = answers["progress"]["probabilities"] + if "on_track" in answers: + out["on_track"] = answers["on_track"]["probabilities"]["on track"] + if "drawn" in answers: + out["drawn"] = answers["drawn"]["choice"] + out["drawn_probs"] = answers["drawn"]["probabilities"] + return out + + +def judge_canvas(agent, image, task: str = "circle", questions=("drawn", "progress", "on_track")) -> Dict: + """Ask the judgement questions about one image (e.g. the clean final canvas, ``Image.fromarray(env. + canvas_pixels())``, with no cursor on it). Used at the end of an episode to label what was actually drawn.""" + from laya.games import paint_judgements + + qs = {k: v for k, v in paint_judgements(task).items() if k in questions} + return summarize_judgements(agent.predict({"image": image}, qs, strict=True)["answers"]) + + class ModelPolicy: """The model's most likely action from ``env.state()`` (two frames plus pen state and recent actions), via - ``predict``. ``last`` keeps the latest answer (probabilities over ``env.actions``) for logging. + ``predict``, with the judgement questions (``laya.games.paint_judgements``) asked about the same state in the + same call. + + ``last`` keeps the latest action answer (probabilities over ``env.actions``, and ``act_probability``, the + checkpoint's act-vs-escalate gate); ``last_judgement`` the latest judgements. With ``judge_stop``, the policy + answers ``DONE`` when the pen is up and the model judges the task complete (P(progress = 4) >= ``stop_at``), + so its own judgement decides when to stop. Load the agent with ``head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN``: ``predict`` runs with - ``strict=True`` and raises rather than silently cut the question.""" + ``strict=True`` and raises rather than silently cut a question.""" - def __init__(self, agent, task: str = "circle"): - self.agent, self.task, self.last, self.provenance, self._questions = agent, task, None, None, {} + def __init__(self, agent, task: str = "circle", judge: bool = True, judge_stop: bool = False, + stop_at: float = 0.5): + self.agent, self.task, self.judge, self.judge_stop, self.stop_at = agent, task, judge, judge_stop, stop_at + self.last, self.last_judgement, self.provenance, self._questions = None, None, None, {} - def __call__(self, env: JSPaintEnv) -> str: - from laya.games import paint_question + def questions(self, env: JSPaintEnv) -> Dict: + from laya.games import paint_judgements, paint_question key = (env.directions, env.step_px) if key not in self._questions: - self._questions[key] = paint_question(self.task, env.directions, env.step_px) - out = self.agent.predict(env.state(), self._questions[key], strict=True) - self.last, self.provenance = out["answers"]["action"], out.get("provenance") + q = paint_question(self.task, env.directions, env.step_px) + if self.judge: + q.update(paint_judgements(self.task)) + self._questions[key] = q + return self._questions[key] + + def __call__(self, env: JSPaintEnv) -> str: + out = self.agent.predict(env.state(), self.questions(env), strict=True) + answers = out["answers"] + self.last, self.provenance = answers["action"], out.get("provenance") + self.last_judgement = summarize_judgements(answers) if self.judge else None + if (self.judge and self.judge_stop and not env.pen + and self.last_judgement["progress_probs"]["4"] >= self.stop_at): + return "DONE" return self.last["choice"] -def model_policy(agent, task: str = "circle") -> ModelPolicy: - return ModelPolicy(agent, task) +def model_policy(agent, task: str = "circle", **kwargs) -> ModelPolicy: + return ModelPolicy(agent, task, **kwargs) def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step=None) -> Dict: @@ -445,4 +488,4 @@ def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step "compass_moves", "TASKS", "TOOLS", "JSPaintServer", "JSPaintEnv", "circle_expert", "random_policy", - "ModelPolicy", "model_policy", "play_episodes", "default_chromium"] + "ModelPolicy", "model_policy", "judge_canvas", "summarize_judgements", "play_episodes", "default_chromium"] diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index 4de28634..31dc3208 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -105,6 +105,47 @@ The discrete actions are built on a mouse-only tool API: `move_mouse(dx, dy)`, ` `JSPaintEnv.call_tool(name, args)` runs them. A tool-calling agent can therefore drive the same environment with free coordinates. +## Judgement questions + +Each step the model also answers three judgement questions (`laya.games.paint_judgements`) about the same state, +in the same `predict` call as the action: + +| Question | Type | Asks | +|---|---|---| +| `progress` | score, 0–4 | How far the canvas is toward the task, from "nothing useful drawn yet" to "the task is complete" | +| `on_track` | choice | "on track" or "off track": could what is drawn so far become the task by continuing? | +| `drawn` | choice | What the canvas shows: circle, oval, arc, line, square, triangle, scribble or nothing | + +`on_track` uses two named options rather than a yes/no (`noul`) question, because on line drawings the model's +yes/no answers were unreliable and named choices were not. + +The runner logs these and the checkpoint's act-vs-escalate gate (`act_probability`) for every step. At the end of +each episode it asks `drawn` about the clean final canvas, with no cursor, and saves the answer as the episode's +`judged_final` label. That label is what lets a failed attempt be relabelled as an example of what was actually +drawn. With `--judge-stop`, the model's own progress judgement ends the episode: it answers `DONE` once the pen is +up and it rates the task complete. + +### Do the judgements track reality? + +[`examples/jspaint_judge_check.py`](https://github.com/r33drichards/laya-vision/blob/main/examples/jspaint_judge_check.py) +asks the judgements every 10 steps along the scripted expert's circles and along random play. It compares them with +true progress: the share of the 36 sectors around the target circle's centre that the pen has passed through. For +zero-shot `thaitea/laya-vision` (revision `f2fe3c1`, 3 episodes each): + +| True progress (expert) | Checks | Judged progress (0–4) | P(on track) | +|---|---|---|---| +| 0 (not drawing yet) | 4 | 1.61 | 0.27 | +| 0–25% | 11 | 1.50 | 0.26 | +| 25–50% | 12 | 1.55 | 0.27 | +| 50–75% | 12 | 1.57 | 0.26 | +| 75–100% | 16 | 1.68 | 0.28 | +| Random policy, all checks | 69 | 1.70 | 0.30 | + +`drawn` works: it labelled the clean final canvas of all 3 expert circles "circle". `progress` and `on_track` do +not work zero-shot. Both are nearly flat, with a rank correlation of 0.40 between true and judged progress over a +narrow range, and random play is rated slightly higher than the expert. The environment knows the true progress, +so these questions can be trained with labels it produces, alongside the actions. + ## The circle verifier `score_circle(pixels)` reads the canvas pixels from the page with `toDataURL`, so the cursor overlay is never diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py index 270c8d49..6c8eb480 100644 --- a/tests/test_paintenv.py +++ b/tests/test_paintenv.py @@ -75,6 +75,22 @@ def test_full_question_fits_raised_budget_without_truncation(): assert n_opts + n_ins + 2 * 64 + 256 <= PAINT_MAX_LEN # two 512px images are 64 tokens each +def test_judgement_questions_and_summary(): + from laya.games import PAINT_SHAPES, paint_judgements + from laya.paintenv import summarize_judgements + + q = paint_judgements("circle") + assert set(q) == {"progress", "on_track", "drawn"} + assert q["progress"]["type"] == "score" and len(q["progress"]["criteria"]) == 5 + assert set(q["on_track"]["criteria"]) == {"on track", "off track"} + assert set(q["drawn"]["criteria"]) == set(PAINT_SHAPES) and "circle" in PAINT_SHAPES and "oval" in PAINT_SHAPES + fake = {"progress": {"score": 2.5, "probabilities": {str(i): 0.2 for i in range(5)}}, + "on_track": {"probabilities": {"on track": 0.7, "off track": 0.3}}, + "drawn": {"choice": "arc", "probabilities": {"arc": 0.6}}} + j = summarize_judgements(fake) + assert j["progress"] == 2.5 and j["on_track"] == 0.7 and j["drawn"] == "arc" + + def test_reset_is_blank_and_observation_is_canvas(env): obs = env.reset(seed=3) assert obs.size == (env.width, env.height) == (512, 512) From cb1b150ecf9b12164b6984cfdc83b45297ad0280 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:12:53 +0000 Subject: [PATCH 09/24] JSPaint training data and Modal jobs: state-based labeller, prepare_paint, paint_eval laya/paintdata.py labels any JSPaint state from the canvas as displayed: the action (finish the target ring from wherever the cursor and ink are, DAgger style, soft over neighbouring compass points) and the judgement questions (progress, on track, drawn). JSPaintEnv.visible_pixels reads the canvas including the stroke in progress (JSPaint commits strokes only on pointerup) and hides the brush preview under the pointer. As a policy the labeller draws a passing circle and stops itself (6/6, verifier 0.95-0.97). env.state() is now {images, context}, the same shape as a training record's images + state_text, so play and training build identical inputs. The training loader (make_item) now takes head_max_len from the checkpoint (processor.laya_head_max_len) instead of a fixed 256, and finetune_long takes max_len / head_max_len overrides that also apply with --init-from and are saved with the new checkpoint, so the ~1,050-token drawing question trains and plays uncut. modal_app.py: prepare_paint (episodes over parallel paint_shard containers, JSPaint pinned at 53be67a, Chromium via playwright) writes /data/vqa/paint_circle; paint_eval plays a checkpoint in JSPaint on an L4. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/paintdata.py | 205 ++++++++++++++++++++++++++++++++++++++++ laya/paintenv.py | 32 +++++-- laya/vlm.py | 1 + laya/vlm_train.py | 8 +- modal_app.py | 181 ++++++++++++++++++++++++++++++++++- tests/test_paintdata.py | 77 +++++++++++++++ tests/test_paintenv.py | 10 +- 7 files changed, 495 insertions(+), 19 deletions(-) create mode 100644 laya/paintdata.py create mode 100644 tests/test_paintdata.py diff --git a/laya/paintdata.py b/laya/paintdata.py new file mode 100644 index 00000000..741e6d18 --- /dev/null +++ b/laya/paintdata.py @@ -0,0 +1,205 @@ +"""Labelled training data for the JSPaint drawing task: actions and the model's own judgements, from any state. + +The scripted ``laya.paintenv.circle_expert`` only works when it is in charge from the first step. Training data +needs labels for the states a learner actually reaches, including ones after its own mistakes, so this module +labels a state from what is on the canvas: + +* **Target circle**: centred on the canvas, radius ``radius_frac`` of the canvas side (the task asks for a circle + about a third of the canvas across). +* **Coverage**: which of ``SECTORS`` angular sectors around the centre hold ink within ``band`` of the ring, read + from the canvas as displayed (``JSPaintEnv.visible_pixels``, which includes the stroke still being drawn). + **Stray ink**: the share of ink farther than that from the ring. +* **Action** (``label_action``): pen down and still on the ring -> the forward move that stays closest to the + radius (``paintenv._around``); pen down but drifted off the ring, or the ring complete -> ``PEN_UP``; pen up + and the ring complete -> ``DONE``; pen up otherwise -> walk to the start of the nearest uncovered gap (so the + clockwise stroke fills it) and press once within half a step of the radius. Stray ink is ignored: it cannot be + erased with the mouse-only actions, so the labeller finishes the circle anyway. +* **Judgements** (``label_judgements``), for ``laya.games.paint_judgements``: ``progress`` 0-4 from coverage (0 no + ring ink, 1 under 40%, 2 under 75%, 3 short of complete, 4 complete); ``on_track`` off track once stray ink is + more than ``STRAY_OFF_TRACK`` of all ink; ``drawn`` nothing / arc / circle when the canvas is only ring ink + (anything else is left unlabelled rather than guessed). + +``play_labelled_episode`` runs one episode under a behaviour policy (the labeller with probability ``1 - eps``, +else a random action, optionally after ``random_prefix`` random steps, which gives off-track and recovery states) +and returns training records in the ``laya.vlm_train.load_jsonl_examples`` format. The action label is always the +labeller's action for the state, not the action the behaviour policy took (DAgger-style), with the probability +spread to the two neighbouring compass points (the moves on either side are nearly as good). +""" +import math +import random +from typing import Dict, List, Optional, Tuple + +import numpy as np + +from laya.circle_verifier import ink_mask +from laya.paintenv import COMPASS, PEN_ACTIONS, _around, _toward + +SECTORS = 36 +STRAY_OFF_TRACK = 0.1 +DIRECTION_SPREAD = 0.1 # target probability on each neighbouring compass point of the labelled move + + +def target_circle(width: int, height: int, radius_frac: float = 0.3) -> Tuple[float, float, float]: + return width / 2.0, height / 2.0, radius_frac * min(width, height) + + +def ring_state(pixels: np.ndarray, circle, band: Optional[float] = None) -> Dict: + """Coverage of the target ring and stray ink, from the true canvas pixels.""" + cx, cy, r = circle + band = band if band is not None else max(4.0, 0.08 * r) + ys, xs = np.nonzero(ink_mask(pixels)) + covered = np.zeros(SECTORS, dtype=bool) + if len(xs) == 0: + return {"covered": covered, "coverage": 0.0, "ink": 0, "stray": 0.0} + d = np.hypot(xs - cx, ys - cy) + on = np.abs(d - r) <= band + ang = np.arctan2(ys[on] - cy, xs[on] - cx) + covered[((ang + np.pi) / (2 * np.pi) * SECTORS).astype(int) % SECTORS] = True + return {"covered": covered, "coverage": float(covered.mean()), "ink": int(len(xs)), + "stray": float(1.0 - on.mean())} + + +def _sector(angle: float) -> int: + return int((angle + math.pi) / (2 * math.pi) * SECTORS) % SECTORS + + +def _gap_start(covered: np.ndarray, pos, circle) -> Tuple[float, float]: + """The ring point where the nearest uncovered run begins, going clockwise (increasing screen angle).""" + cx, cy, r = circle + if not covered.any(): # nothing yet: start at the ring point nearest the cursor + d = math.dist(pos, (cx, cy)) + ux, uy = ((pos[0] - cx) / d, (pos[1] - cy) / d) if d > 1e-6 else (1.0, 0.0) + return cx + r * ux, cy + r * uy + starts = [k for k in range(SECTORS) if not covered[k] and covered[k - 1]] + pts = [] + for k in starts: + a = (k + 0.25) / SECTORS * 2 * math.pi - math.pi # a little into the uncovered sector + pts.append((cx + r * math.cos(a), cy + r * math.sin(a))) + return min(pts, key=lambda p: math.dist(p, pos)) + + +def label_action(env, pixels: np.ndarray, circle) -> str: + """The labeller's action for the environment's current state (see the module docstring).""" + cx, cy, r = circle + ring = ring_state(pixels, circle) + step, pos = env.step_px, env.cursor + complete = bool(ring["covered"].all()) + off_ring = abs(math.dist(pos, (cx, cy)) - r) + if env.pen: + if complete or off_ring > 1.5 * step: + return "PEN_UP" + return _around(env.moves, pos, (cx, cy), r, step) + if complete: + return "DONE" + target = _gap_start(ring["covered"], pos, circle) + if math.dist(pos, target) <= 0.75 * step and off_ring <= 0.5 * step: + return "PEN_DOWN" + return _toward(env.moves, pos, target) + + +def label_judgements(pixels: np.ndarray, circle) -> Dict[str, Optional[str]]: + """Labels for ``paint_judgements``: ``progress`` (level index as a string), ``on_track`` and ``drawn`` (None + when the canvas is not a clean ring or part of one).""" + ring = ring_state(pixels, circle) + cov, complete = ring["coverage"], bool(ring["covered"].all()) + if ring["ink"] == 0 or cov == 0: + progress = 0 + elif complete: + progress = 4 + else: + progress = 1 if cov < 0.4 else 2 if cov < 0.75 else 3 + stray = ring["ink"] > 0 and ring["stray"] > STRAY_OFF_TRACK + if ring["ink"] == 0: + drawn = "nothing" + elif not stray and complete: + drawn = "circle" + elif not stray and cov > 0: + drawn = "arc" + else: + drawn = None + return {"progress": str(progress), "on_track": "off track" if stray else "on track", "drawn": drawn} + + +def direction_target(options: List[str], action: str) -> List[float]: + """A soft target over the action question's options: the labelled move, with ``DIRECTION_SPREAD`` on each + neighbouring compass point among ``options``; one-hot for pen actions.""" + t = [0.0] * len(options) + idx = options.index(action) + if action in PEN_ACTIONS: + t[idx] = 1.0 + return t + moves = [o for o in options if o not in PEN_ACTIONS] + order = sorted(moves, key=COMPASS.index) + i = order.index(action) + for nb in (order[(i - 1) % len(order)], order[(i + 1) % len(order)]): + t[options.index(nb)] += DIRECTION_SPREAD + t[idx] = 1.0 - sum(t) + return t + + +def _question_record(q: Dict) -> Dict: + return {"type": q["type"], "instructions": q["instructions"], "criteria": q["criteria"]} + + +def play_labelled_episode(env, seed: int, rid: str, save_frame, eps: float = 0.0, random_prefix: int = 0, + judge_every: int = 3, task: str = "circle") -> Tuple[List[Dict], Dict]: + """Play one episode and return (training records, episode summary). + + ``save_frame(name, image) -> path`` stores an observation and returns the path the records should reference. + Each step yields an action record; every ``judge_every`` steps also one record per labelled judgement.""" + from laya.games import paint_judgements, paint_question + + rng = random.Random(seed) + circle = target_circle(env.canvas_size, env.canvas_size) + q_act = paint_question(task, env.directions, env.step_px)["action"] + options = list(q_act["criteria"]) + q_judge = paint_judgements(task) + records: List[Dict] = [] + env.reset(seed) + frame_paths: List[str] = [] + + def frame_path(i: int) -> str: + while len(frame_paths) <= i: + k = len(frame_paths) + frame_paths.append(save_frame("%s-%03d" % (rid, k), env._recent_obs[-1])) + return frame_paths[i] + + while not env.done: + t = env.steps + pixels = env.visible_pixels() # includes the stroke in progress (canvas_pixels would not) + now = frame_path(t) + prev = frame_paths[max(0, t - (len(env._recent_obs) - 1))] + base = {"images": [prev, now], "state_text": env.state_text()} + label = label_action(env, pixels, circle) + records.append(dict(base, id="%s-%03d-action" % (rid, t), question=_question_record(q_act), + label=options.index(label), target=direction_target(options, label))) + if t % judge_every == 0: + for name, lab in label_judgements(pixels, circle).items(): + if lab is None: + continue + q = q_judge[name] + crit = q["criteria"] + idx = int(lab) if q["type"] == "score" else list(crit).index(lab) + records.append(dict(base, id="%s-%03d-%s" % (rid, t, name), question=_question_record(q), label=idx)) + if t < random_prefix or rng.random() < eps: + act = rng.choice([a for a in env.actions if a != "DONE"]) + else: + act = label + env.step(act) + if not env.done: + frame_path(env.steps) + summary = {"seed": seed, "eps": eps, "random_prefix": random_prefix, "steps": env.steps, + "records": len(records), "verifier": env.result["score"], + "final": label_judgements(env.visible_pixels(), circle)} + return records, summary + + +class LabellerPolicy: + """``label_action`` as a policy (for checking that the labeller draws a circle on its own).""" + + def __call__(self, env) -> str: + return label_action(env, env.visible_pixels(), target_circle(env.canvas_size, env.canvas_size)) + + +__all__ = ["SECTORS", "target_circle", "ring_state", "label_action", "label_judgements", "direction_target", + "play_labelled_episode", "LabellerPolicy"] diff --git a/laya/paintenv.py b/laya/paintenv.py index 67a06179..6e062e2d 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -295,17 +295,20 @@ def render(self, full_page: bool = False): d.line([x, y - 11, x, y + 11], fill=CURSOR) return img + def state_text(self) -> str: + """The text half of ``state()``: which image is which, the pen state, the step count and recent actions. + Training records store exactly this string (``laya.paintdata``), so play and training see the same input.""" + recent = [t["action"] for t in self.trajectory[-self.history:]] + return ("The first image is the canvas %d steps ago, the second is now. The pen is %s. Step %d of %d. " + "Recent actions, oldest first: %s." % ( + len(self._recent_obs) - 1, "down (drawing)" if self.pen else "up (not drawing)", + self.steps + 1, self.max_steps, " ".join(recent) if recent else "none yet")) + def state(self) -> Dict: """What the model sees this step: ``images`` = [canvas ``frame_gap`` steps ago (the first frame early in an - episode), canvas now], both with the cursor drawn on, plus the pen state, step count and recent actions.""" - recent = [t["action"] for t in self.trajectory[-self.history:]] - return { - "images": [self._recent_obs[0], self._recent_obs[-1]], - "frames": "the first image is the canvas %d steps ago, the second is now" % (len(self._recent_obs) - 1), - "pen": "down (drawing)" if self.pen else "up (not drawing)", - "step": "%d of %d" % (self.steps + 1, self.max_steps), - "recent_actions": " ".join(recent) if recent else "none yet", - } + episode), canvas now], both with the cursor drawn on, and ``context`` = ``state_text()``. The keys match a + training record's (``images`` + ``state_text``, which the loader passes as ``context``).""" + return {"images": [self._recent_obs[0], self._recent_obs[-1]], "context": self.state_text()} def note(self) -> str: return "Task: %s The pen is %s. Step %d of %d." % (TASKS[self.task], "down (drawing)" if self.pen else @@ -320,6 +323,17 @@ def canvas_pixels(self) -> np.ndarray: white = Image.new("RGBA", img.size, (255, 255, 255, 255)) return np.asarray(Image.alpha_composite(white, img).convert("RGB")) + def visible_pixels(self, hide_cursor_px: int = 5) -> np.ndarray: + """The canvas as it looks on screen, as an ``(h, w, 3)`` uint8 array, without our cursor overlay. Unlike + ``canvas_pixels`` this includes a stroke still being drawn: JSPaint keeps the stroke on an overlay and only + commits it to the canvas when the button is released. JSPaint also previews the brush tip under the mouse + pointer, which is not ink, so pixels within ``hide_cursor_px`` of the cursor are painted white.""" + arr = np.array(self.screenshot()) + if hide_cursor_px: + yy, xx = np.ogrid[: arr.shape[0], : arr.shape[1]] + arr[(xx - self.cursor[0]) ** 2 + (yy - self.cursor[1]) ** 2 <= hide_cursor_px ** 2] = 255 + return arr + def close(self) -> None: self.browser.close() self._pw.stop() diff --git a/laya/vlm.py b/laya/vlm.py index 828688f4..2db76fcd 100644 --- a/laya/vlm.py +++ b/laya/vlm.py @@ -917,6 +917,7 @@ def __init__( self.cfg["readout"] = self.processor.laya_readout = self.model.readout self.cfg["option_attention"] = self.model.option_attention self.processor.laya_max_len = self.cfg.get("max_len", 1024) + self.processor.laya_head_max_len = self.cfg.get("head_max_len", 256) self.prep.check(self.processor) self.temperature = self.cfg.get("temperature", [1.0, 1.0, 1.0]) self.temperature_by_options = self.cfg.get("temperature_by_options", {}) diff --git a/laya/vlm_train.py b/laya/vlm_train.py index e4a78af5..b148cc31 100644 --- a/laya/vlm_train.py +++ b/laya/vlm_train.py @@ -120,10 +120,14 @@ def synthetic_examples(n: int = 8, seed: int = 0) -> List[Dict]: def make_item( processor, ex: Dict, rng: random.Random, shuffle: bool = True, max_len: Optional[int] = None, - head_max_len: int = 256, order: Optional[List[int]] = None, + head_max_len: Optional[int] = None, order: Optional[List[int]] = None, ) -> Dict: """Tokenize one example with a random (or the given) option order; the target is permuted to marker order. - ``max_len`` defaults to the agent's (``laya.vlm.build_vlm_inputs``).""" + ``max_len`` and ``head_max_len`` default to the agent's (``laya.vlm.build_vlm_inputs``; the agent leaves them on + the processor as ``laya_max_len`` / ``laya_head_max_len``), so training and evaluation build the same inputs as + ``predict``.""" + if head_max_len is None: + head_max_len = getattr(processor, "laya_head_max_len", 256) k = len(render_options(ex["q"])) if order is None: order = list(range(k)) diff --git a/modal_app.py b/modal_app.py index e1437699..f1991292 100644 --- a/modal_app.py +++ b/modal_app.py @@ -407,6 +407,7 @@ def finetune_long( mix_alpha: float = 0.0, split_edge: int = 0, max_len: int = 0, + head_max_len: int = 0, max_train: int = 0, max_val: int = 0, restart: bool = False, @@ -450,8 +451,10 @@ def finetune_long( * ``split_edge`` > 0 turns on the processor's image splitting (``laya.preprocess``): each image is resized to that longest edge and cut into 512 tiles plus a global view, up to 17 views at 2048 against 1 without. Needs ``preprocess="processor"``. ``max_len`` (0: ``laya.vlm.default_max_len``, 1024 without splitting) - is the sequence cap; both are saved in the checkpoint. Neither applies with ``init_from``, which keeps the - checkpoint's. + is the sequence cap; both are saved in the checkpoint. ``split_edge`` does not apply with ``init_from``, + which keeps the checkpoint's. ``max_len`` and ``head_max_len`` (0: the checkpoint's, 256 for a fresh + head) do: they override the starting checkpoint's and are saved with the new one, so a task whose question + needs more than 256 tokens (the JSPaint drawing question, ~1,050) trains and plays uncut. * ``max_train`` / ``max_val`` > 0 keep the first records per dataset (file order) for a quicker run on a subset; ``max_train`` does not count the ``n_calib`` holdout. * Durability: every ``state_every_min`` minutes (clamped to ``vlm_train.MIN_STATE_MINUTES``, and backed off @@ -495,11 +498,13 @@ def finetune_long( % (max_passes or None, {n: round(epochs * len(train_ex) * probs[n] / sizes[n], 2) for n in names})) if init_from: - agent = VLMAgent(_ckpt_path(init_from), device="cuda") + budgets = {k: v for k, v in (("max_len", max_len), ("head_max_len", head_max_len)) if v} + agent = VLMAgent(_ckpt_path(init_from), device="cuda", **budgets) print("initialised from %s (temperatures %s)" % (init_from, [round(t, 3) for t in agent.temperature])) else: agent = VLMAgent(backbone=backbone, device="cuda", preprocess=preprocess, option_attention=option_attention, - image_split_edge=split_edge, **({"max_len": max_len} if max_len else {})) + image_split_edge=split_edge, **({"max_len": max_len} if max_len else {}), + **({"head_max_len": head_max_len} if head_max_len else {})) print("backbone %s, readout %s, preprocess %s, split_edge %d, max_len %d" % ( agent.cfg["backbone"], agent.model.readout, agent.prep.backend, agent.prep.split_edge, agent.cfg["max_len"])) init_temps = list(agent.temperature) @@ -515,6 +520,7 @@ def finetune_long( log = {"run": run_name, "args": dict(backbone=agent.cfg["backbone"], readout=agent.model.readout, datasets=datasets, val_datasets=val_datasets or datasets, preprocess=agent.prep.backend, split_edge=agent.prep.split_edge, max_len=agent.cfg["max_len"], + head_max_len=agent.cfg.get("head_max_len", 256), max_train=max_train, max_val=max_val, option_attention=agent.model.option_attention, w_ce_schedule=w_ce_schedule, mix=mix_weights, mix_alpha=mix_alpha, @@ -2540,3 +2546,170 @@ def full_eval(model: str, parts: str = ",".join(FULL_EVAL_PARTS), datasets: str print("saved", path, "on laya-checkpoints") if failed: raise SystemExit("full_eval: %s produced no results (see the errors above)" % ", ".join(failed)) + + +# --------------------------------------------------------------------------------------------------------------- +# JSPaint drawing task (laya/paintenv.py, laya/paintdata.py): labelled data from headless Chromium, and play eval. +# JSPaint is cloned at a pinned commit; Chromium comes from `playwright install --with-deps`. +JSPAINT_REPO = "https://github.com/r33drichards/jspaint" +JSPAINT_COMMIT = "53be67ab8c47cc0d2168899e7481bc04839c4c81" +paint_image = _with_local_code( + base_image.pip_install("playwright==1.63.0").run_commands( + "playwright install --with-deps chromium", + "git clone %s /jspaint && cd /jspaint && git checkout %s" % (JSPAINT_REPO, JSPAINT_COMMIT), + ) +) +PAINT_DATASET = "paint_circle" + + +@app.function(image=paint_image, cpu=2, memory=4096, timeout=2 * 60 * 60, volumes={"/data": data_vol}, + retries=modal.Retries(max_retries=2, initial_delay=5.0)) +def paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int = 3) -> list: + """Play the episodes in ``specs`` (``[seed, eps, random_prefix]``) and write ``-.jsonl`` plus + the frames under ``tmp_dir``. Returns one summary per episode.""" + from laya.paintdata import play_labelled_episode + from laya.paintenv import JSPaintEnv, JSPaintServer + + os.makedirs(os.path.join(tmp_dir, "images"), exist_ok=True) + + def save_frame(name, img): + rel = "images/%s.png" % name + img.save(os.path.join(tmp_dir, rel), optimize=True) + return rel + + out, summaries = os.path.join(tmp_dir, "%s-%03d.jsonl" % (split, shard)), [] + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, executable_path=None) as env, \ + open(out + ".part", "w") as f: + for seed, eps, prefix in specs: + recs, summ = play_labelled_episode(env, seed, "%s-%d" % (split, seed), save_frame, eps=eps, + random_prefix=prefix, judge_every=judge_every) + f.writelines(json.dumps(r) + "\n" for r in recs) + summaries.append(summ) + os.rename(out + ".part", out) + data_vol.commit() + return summaries + + +@app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) +def prepare_paint(n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, judge_every: int = 3): + """Write /data/vqa/paint_circle/{train,val}.jsonl + images/: labelled JSPaint circle episodes. + + Each episode runs a behaviour policy in headless Chromium: the state-based labeller (``laya.paintdata``) with + probability ``1 - eps``, else a random action, after ``random_prefix`` purely random steps. ``eps`` is drawn from + (0, 0.1, 0.2, 0.3) and ``random_prefix`` from (0, 0, 10, 30, 60), so the data covers clean circles, noisy ones, + and recoveries from a messy start. Every step is labelled with the labeller's action for that state (soft + over the neighbouring compass points); every ``judge_every`` steps also with progress, on track and what is + drawn. Train and val use disjoint seeds. Episodes are split over ``shards`` containers. + """ + import random + import shutil + from collections import Counter + + final_dir = "/data/vqa/" + PAINT_DATASET + tmp_dir = final_dir + ".tmp" + shutil.rmtree(tmp_dir, ignore_errors=True) + os.makedirs(os.path.join(tmp_dir, "images")) + data_vol.commit() + rng = random.Random(seed) + jobs = [] + for split, n, seed0 in (("train", n_train, seed), ("val", n_val, seed + 1_000_000)): + specs = [[seed0 + i, rng.choice((0.0, 0.1, 0.2, 0.3)), rng.choice((0, 0, 10, 30, 60))] for i in range(n)] + k = max(1, min(shards, n)) + for s in range(k): + if specs[s::k]: + jobs.append((tmp_dir, split, s, specs[s::k], judge_every)) + summaries = {"train": [], "val": []} + for (_, split, _, _, _), res in zip(jobs, paint_shard.starmap(jobs)): + summaries[split].extend(res) + data_vol.reload() + meta = {"source": "JSPaint %s@%s in headless Chromium, state-based circle labeller (laya/paintdata.py)" + % (JSPAINT_REPO, JSPAINT_COMMIT), "judge_every": judge_every} + for split in ("train", "val"): + parts = sorted(fn for fn in os.listdir(tmp_dir) if fn.startswith(split + "-") and fn.endswith(".jsonl")) + kinds = Counter() + with open(os.path.join(tmp_dir, split + ".jsonl"), "w") as out: + for fn in parts: + with open(os.path.join(tmp_dir, fn)) as f: + for line in f: + kinds[json.loads(line)["id"].rsplit("-", 1)[1]] += 1 + out.write(line) + os.remove(os.path.join(tmp_dir, fn)) + eps = summaries[split] + meta[split] = {"episodes": len(eps), "records": dict(kinds), + "labeller_mean_verifier": round(sum(e["verifier"] for e in eps) / max(1, len(eps)), 4), + "mean_steps": round(sum(e["steps"] for e in eps) / max(1, len(eps)), 1)} + print(split, meta[split]) + with open(os.path.join(tmp_dir, "meta.json"), "w") as f: + json.dump(meta, f, indent=2) + with open(os.path.join(tmp_dir, "episodes.json"), "w") as f: + json.dump(summaries, f) + _write_manifest(tmp_dir, {JSPAINT_REPO: JSPAINT_COMMIT}, + dict(n_train=n_train, n_val=n_val, shards=shards, seed=seed, judge_every=judge_every)) + shutil.rmtree(final_dir, ignore_errors=True) + os.rename(tmp_dir, final_dir) + open(os.path.join(final_dir, "_READY"), "w").close() + data_vol.commit() + return meta + + +@app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=3 * 60 * 60, + volumes={"/cache/hf": hf_vol, "/ckpt": ckpt_vol}) +def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int = 5, seed: int = 900_000, + policies: str = "model,expert,random", judge_stop: bool = False, name: str = ""): + """Play the JSPaint circle task with a checkpoint (a run under /ckpt, or a Hub id) and the reference policies. + + The model runs with the raised token budgets (``paintenv.PAINT_HEAD_MAX_LEN`` / ``PAINT_MAX_LEN``) or the + checkpoint's if larger, the full drawing question, two-frame state and the judgement questions each step. + Seeds start at 900,000, disjoint from ``prepare_paint``'s. The summary (verifier score, pass rate, steps, + action mix and per-step judgements) is written to ``/paint_eval/.json`` for a run on the + volume and returned. + """ + from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, + model_policy, play_episodes, random_policy) + from laya.vlm import VLMAgent + + path = _ckpt_path(model) + src = path if os.path.exists(os.path.join(path, "vlm_agent_config.json")) else model + probe = VLMAgent(src, device="cuda") + budgets = dict(head_max_len=max(PAINT_HEAD_MAX_LEN, probe.cfg.get("head_max_len", 256)), + max_len=max(PAINT_MAX_LEN, probe.cfg.get("max_len", 1024))) + del probe + agent = VLMAgent(src, device="cuda", **budgets) + names = [p for p in policies.split(",") if p] + out = {"model": model, "budgets": budgets, "episodes": episodes, "seed": seed, "judge_stop": judge_stop, + "jspaint": JSPAINT_COMMIT, "policies": {}} + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size) as env: + for pname in names: + pol = {"model": lambda: model_policy(agent, judge_stop=judge_stop), "expert": circle_expert, + "random": lambda: random_policy(seed)}[pname]() + steps_log = [] + + def on_step(env, action, pol=pol, pname=pname, steps_log=steps_log): + if pname == "model": + steps_log.append({"seed": env.seed, "step": env.steps, "action": action, + "top_p": max(pol.last["probabilities"].values()), + "act_probability": pol.last["action"]["act_probability"], + **({"judgement": {k: v for k, v in pol.last_judgement.items() + if not k.endswith("_probs")}} if pol.last_judgement else {})}) + + t = time.time() + res = play_episodes(env, pol, episodes, seed=seed, on_step=on_step) + res["seconds"] = round(time.time() - t, 1) + if pname == "model": + res["provenance"] = pol.provenance + res["steps_log"] = steps_log + out["policies"][pname] = res + print("%-7s mean score %.3f pass rate %.2f mean steps %.1f actions %s (%.0fs)" % ( + pname, res["mean_score"], res["pass_rate"], res["mean_steps"], + dict(sorted(res["actions"].items(), key=lambda kv: -kv[1])[:6]), res["seconds"]), flush=True) + if os.path.exists(os.path.join(path, "vlm_agent_config.json")): + d = os.path.join(path, "paint_eval") + os.makedirs(d, exist_ok=True) + fn = os.path.join(d, (name or time.strftime("%Y%m%d-%H%M%S")) + ".json") + with open(fn, "w") as f: + json.dump(out, f, indent=2, default=str) + ckpt_vol.commit() + print("wrote", fn) + return {k: v for k, v in out.items() if k != "policies"} | { + "summary": {p: {k: r[k] for k in ("mean_score", "pass_rate", "mean_steps", "actions")} + for p, r in out["policies"].items()}} diff --git a/tests/test_paintdata.py b/tests/test_paintdata.py new file mode 100644 index 00000000..52cb18f6 --- /dev/null +++ b/tests/test_paintdata.py @@ -0,0 +1,77 @@ +"""The JSPaint labeller's pieces on synthetic canvases: ring coverage, judgement labels and soft direction targets.""" +import math + +import numpy as np +import pytest + +from laya.paintdata import (DIRECTION_SPREAD, SECTORS, direction_target, label_judgements, ring_state, + target_circle) + +PIL = pytest.importorskip("PIL") +from PIL import Image, ImageDraw # noqa: E402 + +S = 512 +CIRCLE = target_circle(S, S) + + +def canvas(draw): + im = Image.new("RGB", (S, S), "white") + draw(ImageDraw.Draw(im)) + return np.asarray(im) + + +def ring_box(c=CIRCLE): + cx, cy, r = c + return [cx - r, cy - r, cx + r, cy + r] + + +def test_blank_canvas(): + px = canvas(lambda d: None) + assert ring_state(px, CIRCLE)["ink"] == 0 + assert label_judgements(px, CIRCLE) == {"progress": "0", "on_track": "on track", "drawn": "nothing"} + + +def test_full_ring_is_complete_circle(): + px = canvas(lambda d: d.ellipse(ring_box(), outline="black", width=4)) + rs = ring_state(px, CIRCLE) + assert rs["covered"].all() and rs["stray"] < 0.01 + assert label_judgements(px, CIRCLE) == {"progress": "4", "on_track": "on track", "drawn": "circle"} + + +@pytest.mark.parametrize("extent,level", [(60, "1"), (200, "2"), (300, "3")]) +def test_partial_arc_progress_levels(extent, level): + px = canvas(lambda d: d.arc(ring_box(), 0, extent, fill="black", width=4)) + lab = label_judgements(px, CIRCLE) + assert lab["progress"] == level and lab["drawn"] == "arc" and lab["on_track"] == "on track" + + +def test_stray_ink_is_off_track_and_unlabelled_shape(): + def draw(d): + d.arc(ring_box(), 0, 90, fill="black", width=4) + d.line([20, 20, 490, 60], fill="black", width=4) # far from the ring + lab = label_judgements(canvas(draw), CIRCLE) + assert lab["on_track"] == "off track" and lab["drawn"] is None + + +def test_ring_elsewhere_does_not_count_as_progress(): + small = (70, 70, 30) # nowhere near the target ring (distances 233-293 px from the centre vs r=154) + px = canvas(lambda d: d.ellipse(ring_box(small), outline="black", width=4)) + lab = label_judgements(px, CIRCLE) + assert lab["progress"] == "0" and lab["on_track"] == "off track" + + +def test_direction_target_spreads_to_neighbours(): + from laya.games import paint_question + + options = list(paint_question()["action"]["criteria"]) + t = direction_target(options, "N") + assert math.isclose(sum(t), 1.0) + assert t[options.index("N")] == pytest.approx(1 - 2 * DIRECTION_SPREAD) + assert t[options.index("NbE")] == pytest.approx(DIRECTION_SPREAD) + assert t[options.index("NbW")] == pytest.approx(DIRECTION_SPREAD) # wraps around past N + pen = direction_target(options, "PEN_DOWN") + assert pen[options.index("PEN_DOWN")] == 1.0 and sum(pen) == 1.0 + + +def test_sector_count(): + assert SECTORS == 36 diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py index 6c8eb480..4edd03be 100644 --- a/tests/test_paintenv.py +++ b/tests/test_paintenv.py @@ -46,15 +46,17 @@ def test_question_covers_actions_and_tools_are_mouse_only(): def test_state_has_two_frames_and_recent_actions(env): env.reset(seed=2) first = env.state() + assert set(first) == {"images", "context"} assert len(first["images"]) == 2 and first["images"][0] is first["images"][1] - assert first["recent_actions"] == "none yet" - for a in ["PEN_DOWN"] + ["E"] * (env.frame_gap + 3): + assert "none yet" in first["context"] and "pen is up" in first["context"] + acts = ["PEN_DOWN"] + ["E"] * (env.frame_gap + 3) + for a in acts: env.step(a) st = env.state() old, now = st["images"] assert old is not now and old.size == now.size == (env.width, env.height) - assert st["recent_actions"].split() == (["PEN_DOWN"] + ["E"] * (env.frame_gap + 3))[-env.history:] - assert st["pen"].startswith("down") + assert ("oldest first: %s." % " ".join(acts[-env.history:])) in st["context"] + assert "pen is down" in st["context"] def test_full_question_fits_raised_budget_without_truncation(): From f149fc639f5833307a79f3f1a8a6e90daa71d3bb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:13:56 +0000 Subject: [PATCH 10/24] modal_app: install git in the JSPaint image before cloning Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- modal_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modal_app.py b/modal_app.py index f1991292..a7b9e85d 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2554,7 +2554,7 @@ def full_eval(model: str, parts: str = ",".join(FULL_EVAL_PARTS), datasets: str JSPAINT_REPO = "https://github.com/r33drichards/jspaint" JSPAINT_COMMIT = "53be67ab8c47cc0d2168899e7481bc04839c4c81" paint_image = _with_local_code( - base_image.pip_install("playwright==1.63.0").run_commands( + base_image.apt_install("git").pip_install("playwright==1.63.0").run_commands( "playwright install --with-deps chromium", "git clone %s /jspaint && cd /jspaint && git checkout %s" % (JSPAINT_REPO, JSPAINT_COMMIT), ) From 78af9df139dd047814c9c10de8fcaf38ff57c1d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 00:16:40 +0000 Subject: [PATCH 11/24] prepare_paint: collect shard results before zipping Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- modal_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modal_app.py b/modal_app.py index a7b9e85d..08456c94 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2619,7 +2619,7 @@ def prepare_paint(n_train: int = 240, n_val: int = 24, shards: int = 32, seed: i if specs[s::k]: jobs.append((tmp_dir, split, s, specs[s::k], judge_every)) summaries = {"train": [], "val": []} - for (_, split, _, _, _), res in zip(jobs, paint_shard.starmap(jobs)): + for (_, split, _, _, _), res in zip(jobs, list(paint_shard.starmap(jobs))): summaries[split].extend(res) data_vol.reload() meta = {"source": "JSPaint %s@%s in headless Chromium, state-based circle labeller (laya/paintdata.py)" From 8347cc8bb5aa25a8bcdcafe001eda5721ded9188 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 02:01:34 +0000 Subject: [PATCH 12/24] JSPaint labeller: a target the model can see A model trained on the first labeller walked past the start and never knew when to press: the labeller always drew one fixed circle centred on the canvas, so where to press was invisible. The labeller now draws the way a person does: press where the cursor is (if a circle fits, else move away from the visible edges), draw clockwise from the circle's top, and read the rest of the target off the drawing (implied_circle: fixed radius 0.18 of the side, top from a known-curvature parabola fit to the topmost ink). Actions and judgement labels are measured against that implied circle. As a policy it draws a passing circle and stops itself (6/6, verifier 0.92-0.94). Data: a third of random-start episodes are messy (the random steps may draw), so off-track states exist; the final canvas of each episode is judged too. prepare_paint takes a dataset --name and refuses to replace an existing one (prepared datasets are create-only). paint_eval adds the labeller as a reference policy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- examples/jspaint_judge_check.py | 30 +++--- laya/paintdata.py | 176 +++++++++++++++++++++----------- modal_app.py | 35 ++++--- tests/test_paintdata.py | 60 ++++++----- 4 files changed, 186 insertions(+), 115 deletions(-) diff --git a/examples/jspaint_judge_check.py b/examples/jspaint_judge_check.py index 40e138dc..65bbb572 100644 --- a/examples/jspaint_judge_check.py +++ b/examples/jspaint_judge_check.py @@ -1,11 +1,12 @@ """Check whether Laya Vision's judgements track what is really happening on the JSPaint canvas. -The scripted circle expert and a random policy play; every --every steps the model answers the judgement questions -(`laya.games.paint_judgements`: progress 0-4, on track or off track, what is drawn) about the same two-frame state -the drawing policy sees. The truth for progress is the share of the 36 angular sectors around the target circle's -centre that the cursor has passed through with the pen down (0 before drawing, 1 for a full turn). Reported: +The state-based labeller (``laya.paintdata.LabellerPolicy``) and a random policy play; every --every steps the +model answers the judgement questions (`laya.games.paint_judgements`: progress 0-4, on track or off track, what is +drawn) about the same two-frame state the drawing policy sees. The truth for progress is the share of the 36 +angular sectors of the circle the drawing implies (``laya.paintdata.implied_circle``) that hold ink (0 before +drawing, 1 for a full ring). Reported: -* how judged progress rises with true progress along the expert's circles (Spearman rank correlation, and the mean +* how judged progress rises with true progress along the labeller's circles (Spearman rank correlation, and the mean judged progress per quarter of true progress); * judged progress and P(on track) for the random policy, which should stay low; * what the model says the clean final canvas shows, per policy. @@ -16,7 +17,6 @@ """ import argparse import json -import math import os import time @@ -24,18 +24,16 @@ from PIL import Image from laya.games import paint_judgements -from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, judge_canvas, - random_policy, summarize_judgements) - -SECTORS = 36 +from laya.paintdata import LabellerPolicy, implied_circle, ring_state +from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, judge_canvas, random_policy, + summarize_judgements) def true_progress(env) -> float: - """Share of the 36 sectors around the canvas centre that pen-down cursor positions have covered.""" - cx, cy = env.width / 2.0, env.height / 2.0 - hit = {int((math.atan2(t["y"] - cy, t["x"] - cx) + math.pi) / (2 * math.pi) * SECTORS) % SECTORS - for t in env.trajectory if t["pen"]} - return len(hit) / SECTORS + """Share of the sectors of the circle the drawing implies that hold ink (0 with nothing drawn).""" + px = env.visible_pixels() + circle = implied_circle(px, None, env.canvas_size) + return 0.0 if circle is None else ring_state(px, circle)["coverage"] def spearman(a, b) -> float: @@ -66,7 +64,7 @@ def main(): rows, finals = [], {} with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size, executable_path=args.chromium) as env: - for name, make in (("expert", lambda: circle_expert()), ("random", lambda: random_policy(args.seed))): + for name, make in (("expert", LabellerPolicy), ("random", lambda: random_policy(args.seed))): policy = make() finals[name] = [] for i in range(args.episodes): diff --git a/laya/paintdata.py b/laya/paintdata.py index 741e6d18..2494a7d1 100644 --- a/laya/paintdata.py +++ b/laya/paintdata.py @@ -4,23 +4,28 @@ needs labels for the states a learner actually reaches, including ones after its own mistakes, so this module labels a state from what is on the canvas: -* **Target circle**: centred on the canvas, radius ``radius_frac`` of the canvas side (the task asks for a circle - about a third of the canvas across). -* **Coverage**: which of ``SECTORS`` angular sectors around the centre hold ink within ``band`` of the ring, read +* **Target circle, read off the canvas**: the circle is drawn clockwise from its top, so any arc on the canvas + implies its circle: radius ``RADIUS_FRAC`` of the canvas side (about a third of the canvas across) and centre + one radius below the topmost ink (``implied_circle``). With nothing drawn yet, the pen's position is the top. + Nothing about the target is invisible: an earlier labeller used a fixed circle centred on the canvas, and a + model trained on it could not see where to press the button and walked past the start. +* **Coverage**: which of ``SECTORS`` angular sectors around that centre hold ink within ``band`` of the ring, read from the canvas as displayed (``JSPaintEnv.visible_pixels``, which includes the stroke still being drawn). **Stray ink**: the share of ink farther than that from the ring. -* **Action** (``label_action``): pen down and still on the ring -> the forward move that stays closest to the - radius (``paintenv._around``); pen down but drifted off the ring, or the ring complete -> ``PEN_UP``; pen up - and the ring complete -> ``DONE``; pen up otherwise -> walk to the start of the nearest uncovered gap (so the - clockwise stroke fills it) and press once within half a step of the radius. Stray ink is ignored: it cannot be - erased with the mouse-only actions, so the labeller finishes the circle anyway. -* **Judgements** (``label_judgements``), for ``laya.games.paint_judgements``: ``progress`` 0-4 from coverage (0 no - ring ink, 1 under 40%, 2 under 75%, 3 short of complete, 4 complete); ``on_track`` off track once stray ink is - more than ``STRAY_OFF_TRACK`` of all ink; ``drawn`` nothing / arc / circle when the canvas is only ring ink - (anything else is left unlabelled rather than guessed). +* **Action** (``label_action``): nothing drawn and the pen up -> press if a circle starting here fits on the + canvas, else move toward the nearest start that fits (away from the edges, which are visible); pen down and on + the ring -> the forward (clockwise) move that stays closest to the radius (``paintenv._around``); pen down but + drifted off the ring, or the ring complete -> ``PEN_UP``; pen up and the ring complete -> ``DONE``; pen up with + a partial ring -> walk to the end of the arc (the start of the uncovered gap, clockwise) and press there. + Stray ink cannot be erased with the mouse-only actions, so the labeller finishes the circle anyway. +* **Judgements** (``label_judgements``), for ``laya.games.paint_judgements``, against the implied circle: + ``progress`` 0-4 from coverage (0 nothing, 1 under 40%, 2 under 75%, 3 short of complete, 4 complete); + ``on_track`` off track once stray ink is more than ``STRAY_OFF_TRACK`` of all ink or the implied circle does + not fit on the canvas; ``drawn`` nothing / arc / circle when the canvas is only ring ink (anything else is left + unlabelled rather than guessed). ``play_labelled_episode`` runs one episode under a behaviour policy (the labeller with probability ``1 - eps``, -else a random action, optionally after ``random_prefix`` random steps, which gives off-track and recovery states) +else a random action, optionally after ``random_prefix`` random pen-up moves that put the cursor somewhere else) and returns training records in the ``laya.vlm_train.load_jsonl_examples`` format. The action label is always the labeller's action for the state, not the action the behaviour policy took (DAgger-style), with the probability spread to the two neighbouring compass points (the moves on either side are nearly as good). @@ -35,16 +40,61 @@ from laya.paintenv import COMPASS, PEN_ACTIONS, _around, _toward SECTORS = 36 +RADIUS_FRAC = 0.18 +TOP_BAND = 8 # px below the topmost ink used to find the top of the circle +MARGIN = 8 # px a circle keeps from the canvas edge STRAY_OFF_TRACK = 0.1 DIRECTION_SPREAD = 0.1 # target probability on each neighbouring compass point of the labelled move +def radius(side: int) -> float: + return RADIUS_FRAC * side + + def target_circle(width: int, height: int, radius_frac: float = 0.3) -> Tuple[float, float, float]: + """A fixed circle centred on the canvas (used by ``examples/jspaint_judge_check.py`` for the old expert).""" return width / 2.0, height / 2.0, radius_frac * min(width, height) +def implied_circle(pixels: np.ndarray, pen_pos=None, side: Optional[int] = None): + """The circle the canvas implies. Its radius is fixed (``radius``) and it is drawn clockwise from its top, so + its top is where the drawing starts: near the topmost ink the ring follows ``y = top + (x - x0)^2 / 2r``, and a + least-squares fit of that parabola (its curvature is known) to the ink within ``TOP_BAND`` px of the topmost + ink gives ``x0`` whichever way the arc runs off. The centre is one radius below. The estimate depends only on + the start of the drawing, so it holds steady while the rest is drawn. With no ink, the circle whose top is + ``pen_pos`` (or None when that is not given).""" + side = side or min(pixels.shape[:2]) + r = radius(side) + ys, xs = np.nonzero(ink_mask(pixels)) + if not len(xs): + if pen_pos is None: + return None + return float(pen_pos[0]), float(pen_pos[1] + r), r + top = ys.min() + band = ys <= top + TOP_BAND + x, y = xs[band].astype(np.float64), ys[band].astype(np.float64) + x0 = float(x.mean()) + if x.max() - x.min() >= 4: + a = 1.0 / (2 * r) + slope, _ = np.polyfit(x, y - a * x * x, 1) # y - a x^2 = -2 a x0 x + c + x0 = float(np.clip(-slope / (2 * a), x.min(), x.max())) + return x0, float(top + 2 + r), r # +2: half the 4 px brush above the path + + +def fits(circle, side: int) -> bool: + cx, cy, r = circle + return MARGIN <= cx - r and cx + r <= side - MARGIN and MARGIN <= cy - r and cy + r <= side - MARGIN + + +def start_point(pos, side: int) -> Tuple[float, float]: + """The nearest cursor position from which a circle drawn clockwise from its top fits on the canvas.""" + r = radius(side) + return (min(max(pos[0], MARGIN + r + 1), side - MARGIN - r - 1), + min(max(pos[1], MARGIN + 1), side - MARGIN - 2 * r - 1)) + + def ring_state(pixels: np.ndarray, circle, band: Optional[float] = None) -> Dict: - """Coverage of the target ring and stray ink, from the true canvas pixels.""" + """Coverage of the ring and stray ink, from the canvas pixels.""" cx, cy, r = circle band = band if band is not None else max(4.0, 0.08 * r) ys, xs = np.nonzero(ink_mask(pixels)) @@ -59,18 +109,11 @@ def ring_state(pixels: np.ndarray, circle, band: Optional[float] = None) -> Dict "stray": float(1.0 - on.mean())} -def _sector(angle: float) -> int: - return int((angle + math.pi) / (2 * math.pi) * SECTORS) % SECTORS - - def _gap_start(covered: np.ndarray, pos, circle) -> Tuple[float, float]: """The ring point where the nearest uncovered run begins, going clockwise (increasing screen angle).""" cx, cy, r = circle - if not covered.any(): # nothing yet: start at the ring point nearest the cursor - d = math.dist(pos, (cx, cy)) - ux, uy = ((pos[0] - cx) / d, (pos[1] - cy) / d) if d > 1e-6 else (1.0, 0.0) - return cx + r * ux, cy + r * uy - starts = [k for k in range(SECTORS) if not covered[k] and covered[k - 1]] + starts = [k for k in range(SECTORS) if not covered[k] and covered[k - 1]] or [ + k for k in range(SECTORS) if not covered[k]] pts = [] for k in starts: a = (k + 0.25) / SECTORS * 2 * math.pi - math.pi # a little into the uncovered sector @@ -78,11 +121,17 @@ def _gap_start(covered: np.ndarray, pos, circle) -> Tuple[float, float]: return min(pts, key=lambda p: math.dist(p, pos)) -def label_action(env, pixels: np.ndarray, circle) -> str: +def label_action(env, pixels: np.ndarray) -> str: """The labeller's action for the environment's current state (see the module docstring).""" + side, step, pos = env.canvas_size, env.step_px, env.cursor + circle = implied_circle(pixels, pos if env.pen else None, side) + if circle is None: # nothing drawn, pen up: press where a circle fits, else move to where one does + target = start_point(pos, side) + if math.dist(pos, target) <= 0.75 * step: + return "PEN_DOWN" + return _toward(env.moves, pos, target) cx, cy, r = circle ring = ring_state(pixels, circle) - step, pos = env.step_px, env.cursor complete = bool(ring["covered"].all()) off_ring = abs(math.dist(pos, (cx, cy)) - r) if env.pen: @@ -97,26 +146,18 @@ def label_action(env, pixels: np.ndarray, circle) -> str: return _toward(env.moves, pos, target) -def label_judgements(pixels: np.ndarray, circle) -> Dict[str, Optional[str]]: - """Labels for ``paint_judgements``: ``progress`` (level index as a string), ``on_track`` and ``drawn`` (None - when the canvas is not a clean ring or part of one).""" +def label_judgements(pixels: np.ndarray) -> Dict[str, Optional[str]]: + """Labels for ``paint_judgements`` against the implied circle: ``progress`` (level index as a string), + ``on_track`` and ``drawn`` (None when the canvas is not a clean ring or part of one).""" + side = min(pixels.shape[:2]) + circle = implied_circle(pixels, None, side) + if circle is None: + return {"progress": "0", "on_track": "on track", "drawn": "nothing"} ring = ring_state(pixels, circle) cov, complete = ring["coverage"], bool(ring["covered"].all()) - if ring["ink"] == 0 or cov == 0: - progress = 0 - elif complete: - progress = 4 - else: - progress = 1 if cov < 0.4 else 2 if cov < 0.75 else 3 - stray = ring["ink"] > 0 and ring["stray"] > STRAY_OFF_TRACK - if ring["ink"] == 0: - drawn = "nothing" - elif not stray and complete: - drawn = "circle" - elif not stray and cov > 0: - drawn = "arc" - else: - drawn = None + progress = 4 if complete else 0 if cov == 0 else 1 if cov < 0.4 else 2 if cov < 0.75 else 3 + stray = ring["stray"] > STRAY_OFF_TRACK or not fits(circle, side) + drawn = None if stray else "circle" if complete else "arc" return {"progress": str(progress), "on_track": "off track" if stray else "on track", "drawn": drawn} @@ -142,15 +183,16 @@ def _question_record(q: Dict) -> Dict: def play_labelled_episode(env, seed: int, rid: str, save_frame, eps: float = 0.0, random_prefix: int = 0, - judge_every: int = 3, task: str = "circle") -> Tuple[List[Dict], Dict]: + judge_every: int = 3, task: str = "circle", messy: bool = False) -> Tuple[List[Dict], Dict]: """Play one episode and return (training records, episode summary). ``save_frame(name, image) -> path`` stores an observation and returns the path the records should reference. - Each step yields an action record; every ``judge_every`` steps also one record per labelled judgement.""" + Each step yields an action record; every ``judge_every`` steps, and once for the final canvas, also one record + per labelled judgement. With ``messy`` the ``random_prefix`` steps may press the button too, leaving stray ink + to judge and recover from; otherwise they are pen-up moves that only relocate the cursor.""" from laya.games import paint_judgements, paint_question rng = random.Random(seed) - circle = target_circle(env.canvas_size, env.canvas_size) q_act = paint_question(task, env.directions, env.step_px)["action"] options = list(q_act["criteria"]) q_judge = paint_judgements(task) @@ -164,33 +206,42 @@ def frame_path(i: int) -> str: frame_paths.append(save_frame("%s-%03d" % (rid, k), env._recent_obs[-1])) return frame_paths[i] + def judge_records(base, pixels, t): + for name, lab in label_judgements(pixels).items(): + if lab is None: + continue + q = q_judge[name] + idx = int(lab) if q["type"] == "score" else list(q["criteria"]).index(lab) + records.append(dict(base, id="%s-%03d-%s" % (rid, t, name), question=_question_record(q), label=idx)) + + def base_state(t): + now = frame_path(t) + prev = frame_paths[max(0, t - (len(env._recent_obs) - 1))] + return {"images": [prev, now], "state_text": env.state_text()} + while not env.done: t = env.steps pixels = env.visible_pixels() # includes the stroke in progress (canvas_pixels would not) - now = frame_path(t) - prev = frame_paths[max(0, t - (len(env._recent_obs) - 1))] - base = {"images": [prev, now], "state_text": env.state_text()} - label = label_action(env, pixels, circle) + base = base_state(t) + label = label_action(env, pixels) records.append(dict(base, id="%s-%03d-action" % (rid, t), question=_question_record(q_act), label=options.index(label), target=direction_target(options, label))) if t % judge_every == 0: - for name, lab in label_judgements(pixels, circle).items(): - if lab is None: - continue - q = q_judge[name] - crit = q["criteria"] - idx = int(lab) if q["type"] == "score" else list(crit).index(lab) - records.append(dict(base, id="%s-%03d-%s" % (rid, t, name), question=_question_record(q), label=idx)) - if t < random_prefix or rng.random() < eps: + judge_records(base, pixels, t) + if t < random_prefix: # relocate the cursor; with messy, stray strokes too + act = rng.choice([a for a in env.actions if a != "DONE"] if messy else list(env.moves)) + elif rng.random() < eps: act = rng.choice([a for a in env.actions if a != "DONE"]) else: act = label env.step(act) if not env.done: frame_path(env.steps) - summary = {"seed": seed, "eps": eps, "random_prefix": random_prefix, "steps": env.steps, + frame_path(env.steps) # the final canvas, judged once more (it is where "complete" and "circle" are true) + judge_records(base_state(env.steps), env.visible_pixels(), env.steps) + summary = {"seed": seed, "eps": eps, "random_prefix": random_prefix, "messy": messy, "steps": env.steps, "records": len(records), "verifier": env.result["score"], - "final": label_judgements(env.visible_pixels(), circle)} + "final": label_judgements(env.visible_pixels())} return records, summary @@ -198,8 +249,9 @@ class LabellerPolicy: """``label_action`` as a policy (for checking that the labeller draws a circle on its own).""" def __call__(self, env) -> str: - return label_action(env, env.visible_pixels(), target_circle(env.canvas_size, env.canvas_size)) + return label_action(env, env.visible_pixels()) -__all__ = ["SECTORS", "target_circle", "ring_state", "label_action", "label_judgements", "direction_target", +__all__ = ["SECTORS", "RADIUS_FRAC", "radius", "implied_circle", "fits", "start_point", "target_circle", + "ring_state", "label_action", "label_judgements", "direction_target", "play_labelled_episode", "LabellerPolicy"] diff --git a/modal_app.py b/modal_app.py index 08456c94..b47c232d 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2559,13 +2559,12 @@ def full_eval(model: str, parts: str = ",".join(FULL_EVAL_PARTS), datasets: str "git clone %s /jspaint && cd /jspaint && git checkout %s" % (JSPAINT_REPO, JSPAINT_COMMIT), ) ) -PAINT_DATASET = "paint_circle" @app.function(image=paint_image, cpu=2, memory=4096, timeout=2 * 60 * 60, volumes={"/data": data_vol}, retries=modal.Retries(max_retries=2, initial_delay=5.0)) def paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int = 3) -> list: - """Play the episodes in ``specs`` (``[seed, eps, random_prefix]``) and write ``-.jsonl`` plus + """Play the episodes in ``specs`` (``[seed, eps, random_prefix, messy]``) and write ``-.jsonl`` plus the frames under ``tmp_dir``. Returns one summary per episode.""" from laya.paintdata import play_labelled_episode from laya.paintenv import JSPaintEnv, JSPaintServer @@ -2580,9 +2579,9 @@ def save_frame(name, img): out, summaries = os.path.join(tmp_dir, "%s-%03d.jsonl" % (split, shard)), [] with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, executable_path=None) as env, \ open(out + ".part", "w") as f: - for seed, eps, prefix in specs: + for seed, eps, prefix, messy in specs: recs, summ = play_labelled_episode(env, seed, "%s-%d" % (split, seed), save_frame, eps=eps, - random_prefix=prefix, judge_every=judge_every) + random_prefix=prefix, judge_every=judge_every, messy=messy) f.writelines(json.dumps(r) + "\n" for r in recs) summaries.append(summ) os.rename(out + ".part", out) @@ -2591,13 +2590,16 @@ def save_frame(name, img): @app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) -def prepare_paint(n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, judge_every: int = 3): - """Write /data/vqa/paint_circle/{train,val}.jsonl + images/: labelled JSPaint circle episodes. +def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, + judge_every: int = 3): + """Write /data/vqa//{train,val}.jsonl + images/: labelled JSPaint circle episodes. Refuses to replace an + existing prepared dataset (they are create-only; ``paint_circle`` was the first, fixed-target labeller). Each episode runs a behaviour policy in headless Chromium: the state-based labeller (``laya.paintdata``) with probability ``1 - eps``, else a random action, after ``random_prefix`` purely random steps. ``eps`` is drawn from (0, 0.1, 0.2, 0.3) and ``random_prefix`` from (0, 0, 10, 30, 60), so the data covers clean circles, noisy ones, - and recoveries from a messy start. Every step is labelled with the labeller's action for that state (soft + and recoveries from another start; a third of the episodes with a random start are ``messy`` (the random steps + may draw, leaving stray ink). Every step is labelled with the labeller's action for that state (soft over the neighbouring compass points); every ``judge_every`` steps also with progress, on track and what is drawn. Train and val use disjoint seeds. Episodes are split over ``shards`` containers. """ @@ -2605,7 +2607,9 @@ def prepare_paint(n_train: int = 240, n_val: int = 24, shards: int = 32, seed: i import shutil from collections import Counter - final_dir = "/data/vqa/" + PAINT_DATASET + final_dir = "/data/vqa/" + name + if os.path.exists(final_dir): + raise SystemExit("%s exists; prepared datasets are create-only, pass a new --name" % final_dir) tmp_dir = final_dir + ".tmp" shutil.rmtree(tmp_dir, ignore_errors=True) os.makedirs(os.path.join(tmp_dir, "images")) @@ -2613,7 +2617,10 @@ def prepare_paint(n_train: int = 240, n_val: int = 24, shards: int = 32, seed: i rng = random.Random(seed) jobs = [] for split, n, seed0 in (("train", n_train, seed), ("val", n_val, seed + 1_000_000)): - specs = [[seed0 + i, rng.choice((0.0, 0.1, 0.2, 0.3)), rng.choice((0, 0, 10, 30, 60))] for i in range(n)] + specs = [] + for i in range(n): + prefix = rng.choice((0, 0, 10, 30, 60)) + specs.append([seed0 + i, rng.choice((0.0, 0.1, 0.2, 0.3)), prefix, bool(prefix) and rng.random() < 1 / 3]) k = max(1, min(shards, n)) for s in range(k): if specs[s::k]: @@ -2645,7 +2652,6 @@ def prepare_paint(n_train: int = 240, n_val: int = 24, shards: int = 32, seed: i json.dump(summaries, f) _write_manifest(tmp_dir, {JSPAINT_REPO: JSPAINT_COMMIT}, dict(n_train=n_train, n_val=n_val, shards=shards, seed=seed, judge_every=judge_every)) - shutil.rmtree(final_dir, ignore_errors=True) os.rename(tmp_dir, final_dir) open(os.path.join(final_dir, "_READY"), "w").close() data_vol.commit() @@ -2655,8 +2661,10 @@ def prepare_paint(n_train: int = 240, n_val: int = 24, shards: int = 32, seed: i @app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=3 * 60 * 60, volumes={"/cache/hf": hf_vol, "/ckpt": ckpt_vol}) def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int = 5, seed: int = 900_000, - policies: str = "model,expert,random", judge_stop: bool = False, name: str = ""): - """Play the JSPaint circle task with a checkpoint (a run under /ckpt, or a Hub id) and the reference policies. + policies: str = "model,labeller,expert,random", judge_stop: bool = False, name: str = ""): + """Play the JSPaint circle task with a checkpoint (a run under /ckpt, or a Hub id) and the reference policies: + ``labeller`` (the state-based labeller the training data comes from), ``expert`` (the older scripted circle + centred on the canvas) and ``random``. The model runs with the raised token budgets (``paintenv.PAINT_HEAD_MAX_LEN`` / ``PAINT_MAX_LEN``) or the checkpoint's if larger, the full drawing question, two-frame state and the judgement questions each step. @@ -2664,6 +2672,7 @@ def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int action mix and per-step judgements) is written to ``/paint_eval/.json`` for a run on the volume and returned. """ + from laya.paintdata import LabellerPolicy from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, model_policy, play_episodes, random_policy) from laya.vlm import VLMAgent @@ -2681,7 +2690,7 @@ def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size) as env: for pname in names: pol = {"model": lambda: model_policy(agent, judge_stop=judge_stop), "expert": circle_expert, - "random": lambda: random_policy(seed)}[pname]() + "labeller": LabellerPolicy, "random": lambda: random_policy(seed)}[pname]() steps_log = [] def on_step(env, action, pol=pol, pname=pname, steps_log=steps_log): diff --git a/tests/test_paintdata.py b/tests/test_paintdata.py index 52cb18f6..095b7a29 100644 --- a/tests/test_paintdata.py +++ b/tests/test_paintdata.py @@ -4,14 +4,14 @@ import numpy as np import pytest -from laya.paintdata import (DIRECTION_SPREAD, SECTORS, direction_target, label_judgements, ring_state, - target_circle) +from laya.paintdata import (DIRECTION_SPREAD, MARGIN, SECTORS, direction_target, fits, implied_circle, + label_judgements, radius, ring_state, start_point) PIL = pytest.importorskip("PIL") from PIL import Image, ImageDraw # noqa: E402 S = 512 -CIRCLE = target_circle(S, S) +R = radius(S) def canvas(draw): @@ -20,44 +20,56 @@ def canvas(draw): return np.asarray(im) -def ring_box(c=CIRCLE): - cx, cy, r = c +def box(cx, cy, r=R): return [cx - r, cy - r, cx + r, cy + r] def test_blank_canvas(): px = canvas(lambda d: None) - assert ring_state(px, CIRCLE)["ink"] == 0 - assert label_judgements(px, CIRCLE) == {"progress": "0", "on_track": "on track", "drawn": "nothing"} + assert implied_circle(px) is None + assert label_judgements(px) == {"progress": "0", "on_track": "on track", "drawn": "nothing"} -def test_full_ring_is_complete_circle(): - px = canvas(lambda d: d.ellipse(ring_box(), outline="black", width=4)) - rs = ring_state(px, CIRCLE) - assert rs["covered"].all() and rs["stray"] < 0.01 - assert label_judgements(px, CIRCLE) == {"progress": "4", "on_track": "on track", "drawn": "circle"} +@pytest.mark.parametrize("cx,cy", [(256, 256), (150, 330), (380, 140)]) +def test_full_ring_anywhere_is_a_complete_circle(cx, cy): + px = canvas(lambda d: d.ellipse(box(cx, cy), outline="black", width=4)) + c = implied_circle(px) + assert abs(c[0] - cx) <= 2 and abs(c[1] - cy) <= 3 # centre one radius below the topmost ink + assert ring_state(px, c)["covered"].all() + assert label_judgements(px) == {"progress": "4", "on_track": "on track", "drawn": "circle"} @pytest.mark.parametrize("extent,level", [(60, "1"), (200, "2"), (300, "3")]) -def test_partial_arc_progress_levels(extent, level): - px = canvas(lambda d: d.arc(ring_box(), 0, extent, fill="black", width=4)) - lab = label_judgements(px, CIRCLE) - assert lab["progress"] == level and lab["drawn"] == "arc" and lab["on_track"] == "on track" +def test_arc_drawn_clockwise_from_the_top(extent, level): + # PIL angles run clockwise from 3 o'clock, so the top is 270 + px = canvas(lambda d: d.arc(box(256, 256), 270, 270 + extent, fill="black", width=4)) + lab = label_judgements(px) + assert lab == {"progress": level, "on_track": "on track", "drawn": "arc"} def test_stray_ink_is_off_track_and_unlabelled_shape(): def draw(d): - d.arc(ring_box(), 0, 90, fill="black", width=4) - d.line([20, 20, 490, 60], fill="black", width=4) # far from the ring - lab = label_judgements(canvas(draw), CIRCLE) + d.arc(box(256, 256), 270, 360, fill="black", width=4) + d.line([60, 420, 470, 470], fill="black", width=4) # far from the ring + lab = label_judgements(canvas(draw)) assert lab["on_track"] == "off track" and lab["drawn"] is None -def test_ring_elsewhere_does_not_count_as_progress(): - small = (70, 70, 30) # nowhere near the target ring (distances 233-293 px from the centre vs r=154) - px = canvas(lambda d: d.ellipse(ring_box(small), outline="black", width=4)) - lab = label_judgements(px, CIRCLE) - assert lab["progress"] == "0" and lab["on_track"] == "off track" +def test_wrong_size_ring_is_off_track(): + px = canvas(lambda d: d.ellipse(box(256, 256, 40), outline="black", width=4)) + assert label_judgements(px)["on_track"] == "off track" + + +def test_arc_whose_circle_cannot_fit_is_off_track(): + px = canvas(lambda d: d.arc(box(256, S - 40), 270, 300, fill="black", width=4)) # top near the bottom edge + assert not fits(implied_circle(px), S) + assert label_judgements(px)["on_track"] == "off track" + + +def test_start_point_keeps_the_circle_on_the_canvas(): + assert start_point((256, 100), S) == (256, 100) + x, y = start_point((500, 500), S) + assert fits((x, y + R, R), S) and y + 2 * R <= S - MARGIN def test_direction_target_spreads_to_neighbours(): From 4a893a61669f89567368eee79eb2adef765afe44 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 03:50:36 +0000 Subject: [PATCH 13/24] JSPaint data: DAgger rounds with a trained checkpoint driving play_labelled_episode takes a driver (a trained ModelPolicy) that takes the step with probability 1 - beta instead of the labeller; every state is still labelled with the labeller's action. prepare_paint --model runs the shards on L4 containers (paint_shard_dagger). paint-circle-v2 draws the first half of the circle correctly, then drifts off at the bottom into states the labeller's data never visits; this collects exactly those states. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/paintdata.py | 14 ++++++++++--- modal_app.py | 52 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/laya/paintdata.py b/laya/paintdata.py index 2494a7d1..77ac28ed 100644 --- a/laya/paintdata.py +++ b/laya/paintdata.py @@ -183,13 +183,18 @@ def _question_record(q: Dict) -> Dict: def play_labelled_episode(env, seed: int, rid: str, save_frame, eps: float = 0.0, random_prefix: int = 0, - judge_every: int = 3, task: str = "circle", messy: bool = False) -> Tuple[List[Dict], Dict]: + judge_every: int = 3, task: str = "circle", messy: bool = False, driver=None, + beta: float = 0.5) -> Tuple[List[Dict], Dict]: """Play one episode and return (training records, episode summary). ``save_frame(name, image) -> path`` stores an observation and returns the path the records should reference. Each step yields an action record; every ``judge_every`` steps, and once for the final canvas, also one record per labelled judgement. With ``messy`` the ``random_prefix`` steps may press the button too, leaving stray ink - to judge and recover from; otherwise they are pen-up moves that only relocate the cursor.""" + to judge and recover from; otherwise they are pen-up moves that only relocate the cursor. + + ``driver(env) -> action`` (e.g. a trained ``paintenv.ModelPolicy``) takes the step with probability ``1 - beta`` + instead of the labeller (DAgger): the episode then visits the states the model itself reaches, including its + mistakes, and every one of them is still labelled with the labeller's action.""" from laya.games import paint_judgements, paint_question rng = random.Random(seed) @@ -232,6 +237,8 @@ def base_state(t): act = rng.choice([a for a in env.actions if a != "DONE"] if messy else list(env.moves)) elif rng.random() < eps: act = rng.choice([a for a in env.actions if a != "DONE"]) + elif driver is not None and rng.random() >= beta: + act = driver(env) else: act = label env.step(act) @@ -239,7 +246,8 @@ def base_state(t): frame_path(env.steps) frame_path(env.steps) # the final canvas, judged once more (it is where "complete" and "circle" are true) judge_records(base_state(env.steps), env.visible_pixels(), env.steps) - summary = {"seed": seed, "eps": eps, "random_prefix": random_prefix, "messy": messy, "steps": env.steps, + summary = {"seed": seed, "eps": eps, "random_prefix": random_prefix, "messy": messy, + "driver": driver is not None, "beta": beta if driver is not None else None, "steps": env.steps, "records": len(records), "verifier": env.result["score"], "final": label_judgements(env.visible_pixels())} return records, summary diff --git a/modal_app.py b/modal_app.py index b47c232d..5af6686a 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2561,13 +2561,9 @@ def full_eval(model: str, parts: str = ",".join(FULL_EVAL_PARTS), datasets: str ) -@app.function(image=paint_image, cpu=2, memory=4096, timeout=2 * 60 * 60, volumes={"/data": data_vol}, - retries=modal.Retries(max_retries=2, initial_delay=5.0)) -def paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int = 3) -> list: - """Play the episodes in ``specs`` (``[seed, eps, random_prefix, messy]``) and write ``-.jsonl`` plus - the frames under ``tmp_dir``. Returns one summary per episode.""" +def _paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int, model: str, beta: float) -> list: from laya.paintdata import play_labelled_episode - from laya.paintenv import JSPaintEnv, JSPaintServer + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, ModelPolicy os.makedirs(os.path.join(tmp_dir, "images"), exist_ok=True) @@ -2576,12 +2572,19 @@ def save_frame(name, img): img.save(os.path.join(tmp_dir, rel), optimize=True) return rel + driver = None + if model: + from laya.vlm import VLMAgent + + agent = VLMAgent(_ckpt_path(model), device="cuda", head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN) + driver = ModelPolicy(agent, judge=False) out, summaries = os.path.join(tmp_dir, "%s-%03d.jsonl" % (split, shard)), [] with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, executable_path=None) as env, \ open(out + ".part", "w") as f: for seed, eps, prefix, messy in specs: recs, summ = play_labelled_episode(env, seed, "%s-%d" % (split, seed), save_frame, eps=eps, - random_prefix=prefix, judge_every=judge_every, messy=messy) + random_prefix=prefix, judge_every=judge_every, messy=messy, + driver=driver, beta=beta) f.writelines(json.dumps(r) + "\n" for r in recs) summaries.append(summ) os.rename(out + ".part", out) @@ -2589,9 +2592,26 @@ def save_frame(name, img): return summaries +@app.function(image=paint_image, cpu=2, memory=4096, timeout=2 * 60 * 60, volumes={"/data": data_vol}, + retries=modal.Retries(max_retries=2, initial_delay=5.0)) +def paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int = 3) -> list: + """Play the episodes in ``specs`` (``[seed, eps, random_prefix, messy]``) with the labeller driving and write + ``-.jsonl`` plus the frames under ``tmp_dir``. Returns one summary per episode.""" + return _paint_shard(tmp_dir, split, shard, specs, judge_every, "", 1.0) + + +@app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=3 * 60 * 60, + volumes={"/data": data_vol, "/cache/hf": hf_vol, "/ckpt": ckpt_vol.read_only()}, + retries=modal.Retries(max_retries=2, initial_delay=5.0)) +def paint_shard_dagger(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int = 3, model: str = "", + beta: float = 0.5) -> list: + """``paint_shard`` with a trained checkpoint driving each step with probability ``1 - beta`` (DAgger).""" + return _paint_shard(tmp_dir, split, shard, specs, judge_every, model, beta) + + @app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, - judge_every: int = 3): + judge_every: int = 3, model: str = "", beta: float = 0.5): """Write /data/vqa//{train,val}.jsonl + images/: labelled JSPaint circle episodes. Refuses to replace an existing prepared dataset (they are create-only; ``paint_circle`` was the first, fixed-target labeller). @@ -2602,6 +2622,10 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int may draw, leaving stray ink). Every step is labelled with the labeller's action for that state (soft over the neighbouring compass points); every ``judge_every`` steps also with progress, on track and what is drawn. Train and val use disjoint seeds. Episodes are split over ``shards`` containers. + + With ``model`` (a run under /ckpt), the checkpoint drives each step with probability ``1 - beta`` (DAgger, on L4 + containers): the data then covers the states the model itself reaches, labelled with what the labeller would + do there. Use a new ``seed`` per round so the episodes differ from the earlier sets'. """ import random import shutil @@ -2624,13 +2648,16 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int k = max(1, min(shards, n)) for s in range(k): if specs[s::k]: - jobs.append((tmp_dir, split, s, specs[s::k], judge_every)) + jobs.append((tmp_dir, split, s, specs[s::k], judge_every) + ((model, beta) if model else ())) summaries = {"train": [], "val": []} - for (_, split, _, _, _), res in zip(jobs, list(paint_shard.starmap(jobs))): + shard_fn = paint_shard_dagger if model else paint_shard + for job, res in zip(jobs, list(shard_fn.starmap(jobs))): + split = job[1] summaries[split].extend(res) data_vol.reload() meta = {"source": "JSPaint %s@%s in headless Chromium, state-based circle labeller (laya/paintdata.py)" - % (JSPAINT_REPO, JSPAINT_COMMIT), "judge_every": judge_every} + % (JSPAINT_REPO, JSPAINT_COMMIT), "judge_every": judge_every, "driver": model or "labeller", + "beta": beta if model else None} for split in ("train", "val"): parts = sorted(fn for fn in os.listdir(tmp_dir) if fn.startswith(split + "-") and fn.endswith(".jsonl")) kinds = Counter() @@ -2651,7 +2678,8 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int with open(os.path.join(tmp_dir, "episodes.json"), "w") as f: json.dump(summaries, f) _write_manifest(tmp_dir, {JSPAINT_REPO: JSPAINT_COMMIT}, - dict(n_train=n_train, n_val=n_val, shards=shards, seed=seed, judge_every=judge_every)) + dict(n_train=n_train, n_val=n_val, shards=shards, seed=seed, judge_every=judge_every, + model=model, beta=beta)) os.rename(tmp_dir, final_dir) open(os.path.join(final_dir, "_READY"), "w").close() data_vol.commit() From 4c53b0f202abb8f3f23ba88054ff924eb45ea2a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 05:17:09 +0000 Subject: [PATCH 14/24] paint_eval: save each episode's final frame and GIF Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- modal_app.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/modal_app.py b/modal_app.py index 5af6686a..912b1e2d 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2696,7 +2696,8 @@ def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int The model runs with the raised token budgets (``paintenv.PAINT_HEAD_MAX_LEN`` / ``PAINT_MAX_LEN``) or the checkpoint's if larger, the full drawing question, two-frame state and the judgement questions each step. - Seeds start at 900,000, disjoint from ``prepare_paint``'s. The summary (verifier score, pass rate, steps, + Seeds start at 900,000, disjoint from ``prepare_paint``'s. Each episode's final frame and a GIF go to + ``/paint_eval//-seed.{png,gif}``. The summary (verifier score, pass rate, steps, action mix and per-step judgements) is written to ``/paint_eval/.json`` for a run on the volume and returned. """ @@ -2715,13 +2716,22 @@ def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int names = [p for p in policies.split(",") if p] out = {"model": model, "budgets": budgets, "episodes": episodes, "seed": seed, "judge_stop": judge_stop, "jspaint": JSPAINT_COMMIT, "policies": {}} - with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size) as env: + on_volume = os.path.exists(os.path.join(path, "vlm_agent_config.json")) + img_dir = os.path.join(path, "paint_eval", name or time.strftime("%Y%m%d-%H%M%S")) + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size, + keep_frames=True) as env: for pname in names: pol = {"model": lambda: model_policy(agent, judge_stop=judge_stop), "expert": circle_expert, "labeller": LabellerPolicy, "random": lambda: random_policy(seed)}[pname]() steps_log = [] def on_step(env, action, pol=pol, pname=pname, steps_log=steps_log): + if env.done and on_volume: # the final frame and the whole episode, for looking at + os.makedirs(img_dir, exist_ok=True) + stem = os.path.join(img_dir, "%s-seed%d" % (pname, env.seed)) + env.frames[-1].save(stem + ".png") + env.frames[0].save(stem + ".gif", save_all=True, append_images=env.frames[1:], duration=80, + loop=0) if pname == "model": steps_log.append({"seed": env.seed, "step": env.steps, "action": action, "top_p": max(pol.last["probabilities"].values()), @@ -2739,7 +2749,7 @@ def on_step(env, action, pol=pol, pname=pname, steps_log=steps_log): print("%-7s mean score %.3f pass rate %.2f mean steps %.1f actions %s (%.0fs)" % ( pname, res["mean_score"], res["pass_rate"], res["mean_steps"], dict(sorted(res["actions"].items(), key=lambda kv: -kv[1])[:6]), res["seconds"]), flush=True) - if os.path.exists(os.path.join(path, "vlm_agent_config.json")): + if on_volume: d = os.path.join(path, "paint_eval") os.makedirs(d, exist_ok=True) fn = os.path.join(d, (name or time.strftime("%Y%m%d-%H%M%S")) + ".json") From eb24fd4892b47d55396de0fda1900b2606efb5d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 05:23:03 +0000 Subject: [PATCH 15/24] Docs: JSPaint training on Modal and the v1-v3 results paint-circle-v3 (one DAgger round on top of the visible-target labeller) draws a closed circle in one stroke and ends the episode itself in 5/5 unseen episodes, mean verifier 0.61, 2/5 passing on roundness; v1 and v2 scored 0. Metrics per episode in docs/paint-circle-metrics.json. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- docs/paint-circle-metrics.json | 141 ++++++++++++++++++++++++++++++++ site-docs/how-to/jspaint-env.md | 53 ++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 docs/paint-circle-metrics.json diff --git a/docs/paint-circle-metrics.json b/docs/paint-circle-metrics.json new file mode 100644 index 00000000..470d568d --- /dev/null +++ b/docs/paint-circle-metrics.json @@ -0,0 +1,141 @@ +{ + "task": "JSPaint circle, mouse-only, 512x512 canvas, 32 directions x 6 px", + "eval": "modal_app.py::paint_eval, 5 episodes, seeds 900000-900004, verifier laya/circle_verifier.py (pass >= 0.7)", + "runs": { + "paint-circle-v1/best": { + "data": "paint_circle (fixed hidden target, superseded)", + "policies": { + "model": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 260.0, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "expert": { + "mean_score": 0.9669000000000001, + "pass_rate": 1.0, + "mean_steps": 180.8, + "scores": [ + 0.9652, + 0.9718, + 0.9662, + 0.9655, + 0.9658 + ] + }, + "random": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 40.6, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } + } + }, + "paint-circle-v2/best": { + "data": "paint_circle_v2", + "policies": { + "model": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 260.0, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "labeller": { + "mean_score": 0.9355399999999999, + "pass_rate": 1.0, + "mean_steps": 98.2, + "scores": [ + 0.9414, + 0.9365, + 0.9323, + 0.9266, + 0.9409 + ] + }, + "expert": { + "mean_score": 0.9669000000000001, + "pass_rate": 1.0, + "mean_steps": 180.8, + "scores": [ + 0.9652, + 0.9718, + 0.9662, + 0.9655, + 0.9658 + ] + }, + "random": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 40.6, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } + } + }, + "paint-circle-v3/best": { + "data": "paint_circle_v2 + paint_circle_dagger1 (v2 driving, beta 0.5)", + "policies": { + "model": { + "mean_score": 0.6131800000000001, + "pass_rate": 0.4, + "mean_steps": 105.2, + "scores": [ + 0.4635, + 0.4997, + 0.6361, + 0.7043, + 0.7623 + ] + }, + "labeller": { + "mean_score": 0.9355399999999999, + "pass_rate": 1.0, + "mean_steps": 98.2, + "scores": [ + 0.9414, + 0.9365, + 0.9323, + 0.9266, + 0.9409 + ] + }, + "random": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 40.6, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } + } + } + } +} \ No newline at end of file diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md index 31dc3208..07d12483 100644 --- a/site-docs/how-to/jspaint-env.md +++ b/site-docs/how-to/jspaint-env.md @@ -185,3 +185,56 @@ With the full question and two-frame memory, the zero-shot checkpoint moves the actions) but never presses the button, and ends each episode with `DONE` within 6–48 steps. Its top option gets about 0.05 probability against 0.03 for uniform, so it is close to guessing. Recognising a circle is not the same as knowing how to draw one: it was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to train it on, as the game checkpoints were trained on expert frames. + +## Training on Modal + +Three jobs in `modal_app.py` train and test the drawing policy: + +1. **`prepare_paint`** generates the labelled episodes in headless Chromium, over parallel containers. JSPaint is + pinned at commit `53be67a`. + - **Labels:** `laya/paintdata.py` labels every state from what is on the canvas. It gives the action to take + there and the three judgement answers. + - **Behaviour:** the labeller drives, with random actions mixed in (probability 0–0.3) and sometimes a random + start. + - **DAgger (`--model `):** a trained checkpoint drives each step with probability `1 - beta` instead, so + the data covers the states the model itself reaches, including its mistakes. + - **Output:** each run writes a new dataset under `/data/vqa/`; existing datasets are never replaced. +2. **`finetune_long`**, with `--head-max-len 1536 --max-len 2560`. The loader now takes `head_max_len` from the + checkpoint, so training builds the same uncut inputs as `predict`, and the value is saved with the new checkpoint. + Batch 16 fits on an A100 40 GB; batch 32 ran out of memory. +3. **`paint_eval`** plays a checkpoint in JSPaint on an L4, alongside the labeller, the older scripted expert and + random play. It saves each episode's final frame and GIF under `/paint_eval//`. + +### The labeller draws a target the model can see + +The first labeller always drew one circle, centred on the canvas. Nothing on screen showed where that circle +should start, so the model trained on it (`paint-circle-v1`) walked straight past the start and never knew when to +press. + +The labeller now draws the way a person does. It presses where the cursor is, if a circle fits there, and draws +clockwise from the circle's top. The rest of the target then follows from the drawing: the circle has a fixed +radius (0.18 of the canvas side) and its top is where the drawing began. As a policy, this labeller completes and +ends 6 of 6 circles by itself, scoring 0.92–0.94. + +### Results + +These are 5 episodes on unseen seeds (900000–900004) with `paint_eval`, so read them as a spot check. The row +data is in `docs/paint-circle-metrics.json`. + +| Checkpoint | Trained on | Mean score | Passed | Steps | +|---|---|---|---|---| +| `paint-circle-v1` | `paint_circle`: 240 episodes, fixed hidden target | 0.00 | 0/5 | 260 | +| `paint-circle-v2` | `paint_circle_v2`: 240 episodes, visible target, 2 passes | 0.00 | 0/5 | 260 | +| `paint-circle-v3` | v2's data plus `paint_circle_dagger1`: 120 episodes with v2 driving half the steps | 0.61 | 2/5 | 105 | +| Labeller (reference) | – | 0.94 | 5/5 | 98 | +| Random (reference) | – | 0.00 | 0/5 | 41 | + +- **v2** presses at once and draws the first half of the circle correctly: E, then SE, then S, then SW. It then + fails to keep turning at the bottom, drifts off the ring and runs into the corner. Its own judgement notices: + P(on track) falls from 1.0 to 0.01 as it leaves the ring. The labeller's data had almost no such states. +- **v3**, after one DAgger round, draws one continuous closed stroke in every episode, releases the button and + answers `DONE` by itself. It judges the result "circle" with progress 3.7–4.0 out of 4. The three misses fail + only on roundness (0.46–0.64; passing needs 0.7): they close up but are lopsided, with a small gap where the + stroke started. + +Each round took about 1.5 hours on Modal: data generation, about an hour of A100 training, then evaluation. From 345209dc1cd09c8c3d29d238717986d2f33df841 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 06:20:59 +0000 Subject: [PATCH 16/24] JSPaint: a square task (goal text only) and paint_eval --task The model and labeller were only ever trained on circles; this lets a checkpoint be asked for a square to see whether it reads the task at all. There is no square verifier yet; the episode score is still the circle verifier's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/games.py | 3 ++- laya/paintenv.py | 2 +- modal_app.py | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/laya/games.py b/laya/games.py index b7ca27a0..1c0bd9ba 100644 --- a/laya/games.py +++ b/laya/games.py @@ -88,7 +88,8 @@ def snake_question() -> Dict: "PEN_DOWN": "press the mouse button to start drawing", "PEN_UP": "release the mouse button to stop drawing", "DONE": "the drawing is finished", } -PAINT_GOALS = {"circle": "draw one round, closed circle, about as big as a third of the canvas height"} +PAINT_GOALS = {"circle": "draw one round, closed circle, about as big as a third of the canvas height", + "square": "draw one closed square with straight sides, about a third of the canvas across"} def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) -> Dict: diff --git a/laya/paintenv.py b/laya/paintenv.py index 6e062e2d..dd425d29 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -69,7 +69,7 @@ def compass_moves(directions: int = 32) -> Dict[str, Tuple[float, float]]: MOVES = compass_moves(32) ACTIONS = tuple(MOVES) + PEN_ACTIONS -TASKS = {"circle": "Draw a circle on the canvas."} +TASKS = {"circle": "Draw a circle on the canvas.", "square": "Draw a square on the canvas."} TOOLS = [ {"name": "move_mouse", "description": "Move the mouse by (dx, dy) canvas pixels. If the button is held down, " diff --git a/modal_app.py b/modal_app.py index 912b1e2d..9c335cfb 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2689,7 +2689,8 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int @app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=3 * 60 * 60, volumes={"/cache/hf": hf_vol, "/ckpt": ckpt_vol}) def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int = 5, seed: int = 900_000, - policies: str = "model,labeller,expert,random", judge_stop: bool = False, name: str = ""): + policies: str = "model,labeller,expert,random", judge_stop: bool = False, name: str = "", + task: str = "circle"): """Play the JSPaint circle task with a checkpoint (a run under /ckpt, or a Hub id) and the reference policies: ``labeller`` (the state-based labeller the training data comes from), ``expert`` (the older scripted circle centred on the canvas) and ``random``. @@ -2714,14 +2715,15 @@ def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int del probe agent = VLMAgent(src, device="cuda", **budgets) names = [p for p in policies.split(",") if p] - out = {"model": model, "budgets": budgets, "episodes": episodes, "seed": seed, "judge_stop": judge_stop, + out = {"model": model, "task": task, "budgets": budgets, "episodes": episodes, "seed": seed, + "judge_stop": judge_stop, "jspaint": JSPAINT_COMMIT, "policies": {}} on_volume = os.path.exists(os.path.join(path, "vlm_agent_config.json")) img_dir = os.path.join(path, "paint_eval", name or time.strftime("%Y%m%d-%H%M%S")) - with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size, + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, task=task, canvas_size=agent.prep.image_size, keep_frames=True) as env: for pname in names: - pol = {"model": lambda: model_policy(agent, judge_stop=judge_stop), "expert": circle_expert, + pol = {"model": lambda: model_policy(agent, task, judge_stop=judge_stop), "expert": circle_expert, "labeller": LabellerPolicy, "random": lambda: random_policy(seed)}[pname]() steps_log = [] From 5f60be2f0028aa601cbdfaa9da52fe64f05494b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 06:48:29 +0000 Subject: [PATCH 17/24] Quick, Draw! doodles as JSPaint drawing tasks laya/quickdraw.py: fetch recognised doodles per category (range requests on the simplified ndjson, CC BY 4.0; data stays on the Modal volume), place a doodle on the canvas starting at the cursor, and DoodleLabeller, which follows it from any state (walk to each stroke's start, press, trace, lift, DONE; lift and return when the pen strays) with judgement labels from the requested doodle. It reproduces human doodles in JSPaint and stops itself. An independent scorer: a small CNN on rasterised doodles for 24 training and 6 held-out categories (DoodleScorer; JSPaintEnv(scorer=...)), trained on doodles disjoint from the episodes'. A local 800-per-category version scores 84% on held-back doodles and 20/24 on the labeller's JSPaint canvases. Task text works for any category (paint_goal, task_text); episodes take any labeller; ModelPolicy caches questions per task. modal_app.py: prepare_paint --categories train (doodle episodes), train_quickdraw_classifier, doodle_eval (one L4 per category, summarised for training vs held-out categories, labeller as ceiling and random as floor). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/games.py | 19 ++- laya/paintdata.py | 41 ++++- laya/paintenv.py | 20 ++- laya/quickdraw.py | 353 ++++++++++++++++++++++++++++++++++++++++ modal_app.py | 182 ++++++++++++++++++++- tests/test_quickdraw.py | 84 ++++++++++ 6 files changed, 673 insertions(+), 26 deletions(-) create mode 100644 laya/quickdraw.py create mode 100644 tests/test_quickdraw.py diff --git a/laya/games.py b/laya/games.py index 1c0bd9ba..f2037f83 100644 --- a/laya/games.py +++ b/laya/games.py @@ -92,6 +92,15 @@ def snake_question() -> Dict: "square": "draw one closed square with straight sides, about a third of the canvas across"} +def paint_goal(task: str) -> str: + """The task sentence for a drawing task: the hand-written goals for ``circle`` / ``square``, otherwise a simple + doodle of the named thing (any Quick, Draw! category, e.g. ``house`` or ``smiley face``).""" + if task in PAINT_GOALS: + return PAINT_GOALS[task] + article = "an" if task[:1].lower() in "aeiou" else "a" + return "draw %s %s as a simple line doodle, about two fifths of the canvas across" % (article, task) + + def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) -> Dict: """The question for ``laya.paintenv.JSPaintEnv``: a paint canvas, a red cursor, and mouse-only actions (a ``step_px`` move toward each of ``directions`` compass points, then pen down / pen up / done).""" @@ -105,7 +114,7 @@ def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) "instructions": "You are using a paint program with only the mouse. You see the canvas a few steps ago " "and now. The white area is the canvas and the red mark is the mouse cursor: a hollow ring " "with a cross means the button is up, a filled dot means it is held down and moving draws a " - "black line. Your task: %s. Which mouse action should you take now?" % PAINT_GOALS[task], + "black line. Your task: %s. Which mouse action should you take now?" % paint_goal(task), "criteria": {**moves, **PAINT_PEN}, }} @@ -118,7 +127,7 @@ def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) } -def paint_judgements(task: str = "circle") -> Dict: +def paint_judgements(task: str = "circle", shapes: Optional[Dict[str, str]] = None) -> Dict: """The judgement questions asked with ``paint_question`` each step (same state, same ``predict`` call): * ``progress`` (``score``, 0-4): how far the canvas is toward the task, the model's own dense signal; @@ -126,8 +135,10 @@ def paint_judgements(task: str = "circle") -> Dict: line drawings the model's yes/no (``noul``) answers were unreliable while named choices were not; * ``drawn`` (``choice`` over ``PAINT_SHAPES``): what the canvas shows now, whatever the task. At the end of an episode this labels what was actually drawn, so a failed attempt can be relabelled as an example of that. + ``shapes`` replaces ``PAINT_SHAPES`` as its options (e.g. the Quick, Draw! training categories plus + ``nothing``). """ - goal = PAINT_GOALS[task] + goal = paint_goal(task) return { "progress": { "type": "score", @@ -144,7 +155,7 @@ def paint_judgements(task: str = "circle") -> Dict: "drawn": { "type": "choice", "instructions": "What is drawn on this white canvas?", - "criteria": dict(PAINT_SHAPES), + "criteria": dict(shapes or PAINT_SHAPES), }, } diff --git a/laya/paintdata.py b/laya/paintdata.py index 77ac28ed..7d69c9a0 100644 --- a/laya/paintdata.py +++ b/laya/paintdata.py @@ -184,7 +184,7 @@ def _question_record(q: Dict) -> Dict: def play_labelled_episode(env, seed: int, rid: str, save_frame, eps: float = 0.0, random_prefix: int = 0, judge_every: int = 3, task: str = "circle", messy: bool = False, driver=None, - beta: float = 0.5) -> Tuple[List[Dict], Dict]: + beta: float = 0.5, labeller=None) -> Tuple[List[Dict], Dict]: """Play one episode and return (training records, episode summary). ``save_frame(name, image) -> path`` stores an observation and returns the path the records should reference. @@ -194,15 +194,21 @@ def play_labelled_episode(env, seed: int, rid: str, save_frame, eps: float = 0.0 ``driver(env) -> action`` (e.g. a trained ``paintenv.ModelPolicy``) takes the step with probability ``1 - beta`` instead of the labeller (DAgger): the episode then visits the states the model itself reaches, including its - mistakes, and every one of them is still labelled with the labeller's action.""" - from laya.games import paint_judgements, paint_question + mistakes, and every one of them is still labelled with the labeller's action. + ``labeller`` supplies the labels: ``reset(env)`` after the environment resets, ``action(env, pixels)``, + ``judgements(pixels)`` and ``questions(task)`` (the judgement questions). ``CircleLabeller`` by default; + ``laya.quickdraw.DoodleLabeller`` follows a human Quick, Draw! doodle.""" + from laya.games import paint_question + + labeller = labeller or CircleLabeller() rng = random.Random(seed) q_act = paint_question(task, env.directions, env.step_px)["action"] options = list(q_act["criteria"]) - q_judge = paint_judgements(task) + q_judge = labeller.questions(task) records: List[Dict] = [] env.reset(seed) + labeller.reset(env) frame_paths: List[str] = [] def frame_path(i: int) -> str: @@ -212,7 +218,7 @@ def frame_path(i: int) -> str: return frame_paths[i] def judge_records(base, pixels, t): - for name, lab in label_judgements(pixels).items(): + for name, lab in labeller.judgements(pixels).items(): if lab is None: continue q = q_judge[name] @@ -228,7 +234,7 @@ def base_state(t): t = env.steps pixels = env.visible_pixels() # includes the stroke in progress (canvas_pixels would not) base = base_state(t) - label = label_action(env, pixels) + label = labeller.action(env, pixels) records.append(dict(base, id="%s-%03d-action" % (rid, t), question=_question_record(q_act), label=options.index(label), target=direction_target(options, label))) if t % judge_every == 0: @@ -249,10 +255,29 @@ def base_state(t): summary = {"seed": seed, "eps": eps, "random_prefix": random_prefix, "messy": messy, "driver": driver is not None, "beta": beta if driver is not None else None, "steps": env.steps, "records": len(records), "verifier": env.result["score"], - "final": label_judgements(env.visible_pixels())} + "final": labeller.judgements(env.visible_pixels())} return records, summary +class CircleLabeller: + """The circle labeller (``label_action`` / ``label_judgements``) in the interface ``play_labelled_episode`` + takes.""" + + def reset(self, env) -> None: + pass + + def action(self, env, pixels: np.ndarray) -> str: + return label_action(env, pixels) + + def judgements(self, pixels: np.ndarray) -> Dict[str, Optional[str]]: + return label_judgements(pixels) + + def questions(self, task: str) -> Dict: + from laya.games import paint_judgements + + return paint_judgements(task) + + class LabellerPolicy: """``label_action`` as a policy (for checking that the labeller draws a circle on its own).""" @@ -262,4 +287,4 @@ def __call__(self, env) -> str: __all__ = ["SECTORS", "RADIUS_FRAC", "radius", "implied_circle", "fits", "start_point", "target_circle", "ring_state", "label_action", "label_judgements", "direction_target", - "play_labelled_episode", "LabellerPolicy"] + "play_labelled_episode", "CircleLabeller", "LabellerPolicy"] diff --git a/laya/paintenv.py b/laya/paintenv.py index dd425d29..a3d0527a 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -71,6 +71,11 @@ def compass_moves(directions: int = 32) -> Dict[str, Tuple[float, float]]: ACTIONS = tuple(MOVES) + PEN_ACTIONS TASKS = {"circle": "Draw a circle on the canvas.", "square": "Draw a square on the canvas."} + +def task_text(task: str) -> str: + """One sentence naming the task; any Quick, Draw! category works (``laya.quickdraw``).""" + return TASKS.get(task) or "Draw %s %s on the canvas." % ("an" if task[:1].lower() in "aeiou" else "a", task) + TOOLS = [ {"name": "move_mouse", "description": "Move the mouse by (dx, dy) canvas pixels. If the button is held down, " "this draws a brush stroke along the way.", @@ -129,9 +134,9 @@ class JSPaintEnv: def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: int = 32, max_steps: int = 260, reward: str = "terminal", headless: bool = True, executable_path: Optional[str] = None, viewport: Tuple[int, int] = (900, 720), keep_frames: bool = False, canvas_size: int = 512, - frame_gap: int = 8, history: int = 12): - if task not in TASKS: - raise ValueError("unknown task %r (one of %s)" % (task, ", ".join(TASKS))) + frame_gap: int = 8, history: int = 12, scorer=None): + if not task: + raise ValueError("task must name what to draw") if reward not in ("terminal", "shaped"): raise ValueError("reward must be 'terminal' or 'shaped'") from playwright.sync_api import sync_playwright @@ -139,6 +144,7 @@ def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: self.url, self.task, self.step_px, self.max_steps, self.reward_mode = url, task, step_px, max_steps, reward self.keep_frames, self.directions = keep_frames, directions self.frame_gap, self.history = frame_gap, history + self.scorer = scorer # scorer(pixels, task) -> {"score", "passed", ...}; None: the circle verifier self._recent_obs: deque = deque(maxlen=frame_gap + 1) self.moves = compass_moves(directions) self.actions = tuple(self.moves) + PEN_ACTIONS @@ -230,8 +236,12 @@ def step(self, action: str): return obs, reward, self.done, info def verify(self) -> Dict: + """The episode's grade: ``scorer`` if one was given (e.g. ``laya.quickdraw.DoodleScorer`` for any + category), else the circle verifier.""" from laya.circle_verifier import score_circle + if self.scorer is not None: + return self.scorer(self.canvas_pixels(), self.task) return score_circle(self.canvas_pixels()) # -- mouse-only tool API ------------------------------------------------------------------------------------ @@ -311,7 +321,7 @@ def state(self) -> Dict: return {"images": [self._recent_obs[0], self._recent_obs[-1]], "context": self.state_text()} def note(self) -> str: - return "Task: %s The pen is %s. Step %d of %d." % (TASKS[self.task], "down (drawing)" if self.pen else + return "Task: %s The pen is %s. Step %d of %d." % (task_text(self.task), "down (drawing)" if self.pen else "up (not drawing)", self.steps + 1, self.max_steps) def canvas_pixels(self) -> np.ndarray: @@ -453,7 +463,7 @@ def __init__(self, agent, task: str = "circle", judge: bool = True, judge_stop: def questions(self, env: JSPaintEnv) -> Dict: from laya.games import paint_judgements, paint_question - key = (env.directions, env.step_px) + key = (self.task, env.directions, env.step_px) if key not in self._questions: q = paint_question(self.task, env.directions, env.step_px) if self.judge: diff --git a/laya/quickdraw.py b/laya/quickdraw.py new file mode 100644 index 00000000..7a09f00f --- /dev/null +++ b/laya/quickdraw.py @@ -0,0 +1,353 @@ +"""Human doodles from Google's Quick, Draw! as drawing tasks for the JSPaint environment. + +`Quick, Draw! `_ has about 50 million doodles in 345 +categories, each stored as the pen strokes a person drew, in order (CC BY 4.0). This module turns them into tasks +for ``laya.paintenv.JSPaintEnv`` ("draw a house"): + +* ``fetch_drawings`` reads the first recognised doodles of a category from the simplified ndjson files (coordinates + 0-255, strokes simplified with Ramer-Douglas-Peucker), with an HTTP range request so a category costs a few + megabytes, not the whole file. +* ``doodle_strokes`` scales a doodle to ``DOODLE_FRAC`` of the canvas, starts it where the cursor is (shifted only + as far as needed to fit on the canvas, which is visible), and resamples every stroke to the environment's step. +* ``DoodleLabeller`` follows that doodle from any state: walk to the start of the next stroke, press, trace it with + the compass moves, release, and ``DONE`` after the last stroke. If the pen strays, it lifts and returns to where + the stroke left off, so it can label the states a model reaches after its own mistakes (DAgger). Its judgement + labels come from the requested doodle: progress is the share of the doodle's length drawn, on track means little + ink lies away from the doodle's strokes, and ``drawn`` is the category once the doodle is complete. +* ``render_strokes`` / ``to_bitmap`` rasterise doodles and canvases the same way for the independent scorer + (``modal_app.py::train_quickdraw_classifier``), which grades a drawing by the probability a separately trained + classifier gives the requested category, so the model is never graded by its own judgement. + +The episode's doodle is not shown to the model: it only gets the category name, as a person would. +""" +import json +import math +import urllib.parse +import urllib.request +from typing import Dict, List, Optional, Sequence, Tuple + +import numpy as np + +from laya.circle_verifier import ink_mask +from laya.paintenv import _toward + +BASE_URL = "https://storage.googleapis.com/quickdraw_dataset/full/simplified/%s.ndjson" +TRAIN_CATEGORIES = ( + "circle", "square", "triangle", "line", "zigzag", "star", "hexagon", "octagon", "diamond", "moon", "sun", + "cloud", "house", "tree", "fish", "cup", "eye", "envelope", "umbrella", "key", "apple", "mountain", + "smiley face", "hourglass", +) +HELDOUT_CATEGORIES = ("ladder", "stop sign", "cat", "flower", "lightning", "hat") +DOODLE_FRAC = 0.45 # the doodle's longer side, as a share of the canvas side +MAX_STROKES = 8 +MARGIN = 8 + + +def fetch_drawings(category: str, n: int, max_bytes: int = 4_000_000, skip: int = 0, + max_strokes: int = MAX_STROKES) -> List[Dict]: + """The first ``n`` recognised doodles of ``category`` with at most ``max_strokes`` strokes, after skipping + ``skip`` of them (so disjoint sets can be drawn from the same file). Each is ``{"key_id", "strokes": [[(x, y), + ...], ...]}`` in the dataset's 0-255 coordinates.""" + url = BASE_URL % urllib.parse.quote(category) + req = urllib.request.Request(url, headers={"Range": "bytes=0-%d" % (max_bytes - 1)}) + with urllib.request.urlopen(req, timeout=120) as r: + text = r.read().decode("utf-8", errors="ignore") + out, seen = [], 0 + for line in text.split("\n")[:-1]: # the last line may be cut by the range + d = json.loads(line) + if not d.get("recognized") or len(d["drawing"]) > max_strokes: + continue + seen += 1 + if seen <= skip: + continue + out.append({"key_id": d["key_id"], "strokes": [list(zip(xs, ys)) for xs, ys in d["drawing"]]}) + if len(out) >= n: + break + return out + + +def _resample(points: Sequence[Tuple[float, float]], spacing: float) -> List[Tuple[float, float]]: + """Points every ``spacing`` px along a polyline (always keeping its ends).""" + pts = [tuple(map(float, p)) for p in points] + if len(pts) == 1: + return pts + out, carry = [pts[0]], 0.0 + for a, b in zip(pts, pts[1:]): + seg = math.dist(a, b) + d = spacing - carry + while d <= seg: + t = d / seg + out.append((a[0] + t * (b[0] - a[0]), a[1] + t * (b[1] - a[1]))) + d += spacing + carry = seg - (d - spacing) + if math.dist(out[-1], pts[-1]) > 1e-6: + out.append(pts[-1]) + return out + + +def doodle_strokes(strokes: Sequence[Sequence[Tuple[float, float]]], side: int, start: Tuple[float, float], + step: float) -> List[List[Tuple[float, float]]]: + """Canvas-space strokes: the doodle scaled so its longer side is ``DOODLE_FRAC * side``, placed so its first + point is at ``start`` (shifted as little as needed to keep it ``MARGIN`` px inside the canvas), and resampled + every ``step`` px.""" + pts = np.array([p for s in strokes for p in s], dtype=np.float64) + lo, hi = pts.min(0), pts.max(0) + scale = DOODLE_FRAC * side / max(1.0, float((hi - lo).max())) + first = np.array(strokes[0][0], dtype=np.float64) + off = np.array(start, dtype=np.float64) - first * scale + blo, bhi = lo * scale + off, hi * scale + off + off += np.maximum(0.0, MARGIN - blo) - np.maximum(0.0, bhi - (side - MARGIN)) + return [_resample([tuple(np.array(p) * scale + off) for p in s], step) for s in strokes] + + +def stroke_length(stroke: Sequence[Tuple[float, float]]) -> float: + return sum(math.dist(a, b) for a, b in zip(stroke, stroke[1:])) + + +class DoodleLabeller: + """Follows one human doodle (``play_labelled_episode``'s labeller interface). ``drawing`` is a + ``fetch_drawings`` item; it is placed on the canvas at ``reset`` from where the cursor starts.""" + + def __init__(self, category: str, drawing: Dict, shapes: Optional[Dict[str, str]] = None, + band: float = 10.0, stray_off_track: float = 0.15): + self.category, self.drawing, self.band, self.stray_off_track = category, drawing, band, stray_off_track + self.shapes = shapes + + def reset(self, env) -> None: + self.env = env + self.strokes = doodle_strokes(self.drawing["strokes"], env.canvas_size, env.cursor, env.step_px) + self.i, self.j = 0, 0 # next stroke, next point in it + self.lengths = [stroke_length(s) for s in self.strokes] + self.total = max(1e-6, sum(self.lengths)) + self._target_mask = None + + # -- progress along the doodle ----------------------------------------------------------------------------- + def _advance(self, env) -> None: + """Move the pointer past points the pen (down) has reached; start the next stroke once a finished one is + released.""" + if self.i >= len(self.strokes): + return + s, step = self.strokes[self.i], env.step_px + if env.pen: + window = range(self.j, min(len(s), self.j + 4)) + reached = [k for k in window if math.dist(env.cursor, s[k]) <= 0.8 * step] + if reached: + self.j = reached[-1] + 1 + elif self.j >= len(s): + self.i, self.j = self.i + 1, 0 + + def drawn_fraction(self) -> float: + done = sum(self.lengths[: self.i]) + if self.i < len(self.strokes) and self.j > 0: + s = self.strokes[self.i] + done += stroke_length(s[: min(self.j, len(s))]) + return min(1.0, done / self.total) + + @property + def complete(self) -> bool: + return self.i >= len(self.strokes) + + # -- labeller interface ------------------------------------------------------------------------------------ + def action(self, env, pixels: np.ndarray) -> str: + self._advance(env) + step, pos = env.step_px, env.cursor + if self.complete: + return "PEN_UP" if env.pen else "DONE" + s = self.strokes[self.i] + if env.pen: + if self.j >= len(s): + return "PEN_UP" + if math.dist(pos, s[self.j]) > 3 * step: # strayed: lift, then come back to where the stroke left off + return "PEN_UP" + return _toward(env.moves, pos, s[self.j]) + target = s[min(max(self.j - 1, 0), len(s) - 1)] # the stroke's start, or where it was left off + if math.dist(pos, target) <= 0.75 * step: + return "PEN_DOWN" + return _toward(env.moves, pos, target) + + def _stray(self, pixels: np.ndarray) -> float: + """Share of the ink farther than ``band`` px from every stroke of the doodle.""" + ink = ink_mask(pixels) + if not ink.any(): + return 0.0 + if self._target_mask is None or self._target_mask.shape != ink.shape: + from PIL import Image, ImageDraw + + im = Image.new("L", (ink.shape[1], ink.shape[0]), 0) + d = ImageDraw.Draw(im) + for s in self.strokes: + if len(s) > 1: + d.line(s, fill=255, width=int(2 * self.band)) + d.ellipse([s[0][0] - self.band, s[0][1] - self.band, s[0][0] + self.band, s[0][1] + self.band], + fill=255) + self._target_mask = np.asarray(im) > 0 + return float((ink & ~self._target_mask).sum() / ink.sum()) + + def judgements(self, pixels: np.ndarray) -> Dict[str, Optional[str]]: + frac = self.drawn_fraction() + ink = bool(ink_mask(pixels).any()) + stray = ink and self._stray(pixels) > self.stray_off_track + if self.complete: + progress = 4 + elif frac == 0 or not ink: + progress = 0 + else: + progress = 1 if frac < 0.4 else 2 if frac < 0.75 else 3 + if not ink: + drawn = "nothing" + elif self.complete and not stray and self.shapes and self.category in self.shapes: + drawn = self.category + else: + drawn = None + return {"progress": str(progress), "on_track": "off track" if stray else "on track", "drawn": drawn} + + def questions(self, task: str) -> Dict: + from laya.games import paint_judgements + + return paint_judgements(task, shapes=self.shapes) + + +def drawn_options(categories: Sequence[str] = TRAIN_CATEGORIES) -> Dict[str, str]: + """``drawn`` question options for doodle tasks: the categories, plus ``nothing``.""" + opts = {c: "a doodle of %s %s" % ("an" if c[0] in "aeiou" else "a", c) for c in categories} + opts["nothing"] = "nothing, the canvas is blank" + return opts + + +# -- rasterising for the independent classifier -------------------------------------------------------------------- +BITMAP = 28 + + +def to_bitmap(mask: np.ndarray, size: int = BITMAP) -> np.ndarray: + """A boolean ink mask cropped to its ink, padded to a square with a 10% border, and resized to ``size`` px: + ``(size, size)`` float32 in [0, 1]. Blank masks give zeros.""" + from PIL import Image + + ys, xs = np.nonzero(mask) + if not len(xs): + return np.zeros((size, size), np.float32) + y0, y1, x0, x1 = ys.min(), ys.max() + 1, xs.min(), xs.max() + 1 + h, w = y1 - y0, x1 - x0 + side = int(max(h, w) * 1.2) + 2 + sq = np.zeros((side, side), np.uint8) + oy, ox = (side - h) // 2, (side - w) // 2 + sq[oy:oy + h, ox:ox + w] = mask[y0:y1, x0:x1] * 255 + im = Image.fromarray(sq).resize((size, size), Image.BILINEAR) + return np.asarray(im, np.float32) / 255.0 + + +def render_strokes(strokes: Sequence[Sequence[Tuple[float, float]]], canvas: int = 256, + width: int = 4) -> np.ndarray: + """A 0-255 doodle drawn as the environment would draw it at ``DOODLE_FRAC`` of a 512 canvas: an ink mask.""" + from PIL import Image, ImageDraw + + im = Image.new("L", (canvas, canvas), 0) + d = ImageDraw.Draw(im) + for s in strokes: + if len(s) > 1: + d.line([tuple(p) for p in s], fill=255, width=width, joint="curve") + else: + x, y = s[0] + d.ellipse([x - width / 2, y - width / 2, x + width / 2, y + width / 2], fill=255) + return np.asarray(im) > 0 + + +def canvas_bitmap(pixels: np.ndarray) -> np.ndarray: + """The classifier's input for a canvas (RGB pixels).""" + return to_bitmap(ink_mask(pixels)) + + +def make_classifier(n_classes: int): + """A small CNN over ``BITMAP`` x ``BITMAP`` doodle bitmaps (the independent scorer).""" + import torch.nn as nn + + return nn.Sequential( + nn.Conv2d(1, 32, 3, padding=1), nn.ReLU(), nn.Conv2d(32, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.Conv2d(64, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Flatten(), nn.Dropout(0.3), nn.Linear(64 * 7 * 7, 256), nn.ReLU(), nn.Dropout(0.3), + nn.Linear(256, n_classes)) + + +def classifier_dataset(categories: Sequence[str], n_per: int, skip: int = 500, seed: int = 0): + """Bitmaps and labels for the classifier: ``n_per`` doodles per category after the first ``skip`` (which the + drawing episodes use), each rasterised with a random stroke width (3-6 px on a 256 canvas).""" + import random + + rng = random.Random(seed) + xs, ys = [], [] + for k, c in enumerate(categories): + for d in fetch_drawings(c, n_per, max_bytes=12_000_000, skip=skip): + xs.append(to_bitmap(render_strokes(d["strokes"], width=rng.randint(3, 6)))) + ys.append(k) + return np.stack(xs).astype(np.float32), np.array(ys, dtype=np.int64) + + +def train_classifier(categories: Sequence[str], n_per: int = 2000, epochs: int = 8, device: str = "cpu", + seed: int = 0, val_frac: float = 0.1) -> Tuple[Dict, Dict]: + """Train ``make_classifier`` on ``classifier_dataset``. Returns (checkpoint dict, metrics with held-back + top-1 accuracy overall and per category).""" + import torch + + torch.manual_seed(seed) + x, y = classifier_dataset(categories, n_per, seed=seed) + perm = np.random.default_rng(seed).permutation(len(y)) + n_val = int(len(y) * val_frac) + va, tr = perm[:n_val], perm[n_val:] + net = make_classifier(len(categories)).to(device) + opt = torch.optim.Adam(net.parameters(), lr=1e-3) + xt, yt = torch.from_numpy(x[tr])[:, None], torch.from_numpy(y[tr]) + for ep in range(epochs): + net.train() + order = torch.randperm(len(yt)) + for b in range(0, len(order), 256): + idx = order[b:b + 256] + loss = torch.nn.functional.cross_entropy(net(xt[idx].to(device)), yt[idx].to(device)) + opt.zero_grad() + loss.backward() + opt.step() + net.eval() + with torch.no_grad(): + pred = net(torch.from_numpy(x[va])[:, None].to(device)).argmax(-1).cpu().numpy() + per = {c: float((pred[y[va] == k] == k).mean()) for k, c in enumerate(categories) if (y[va] == k).any()} + metrics = {"n_train": int(len(tr)), "n_val": int(n_val), "val_accuracy": float((pred == y[va]).mean()), + "per_category": per, "epochs": epochs, "n_per": n_per} + ckpt = {"categories": list(categories), "state_dict": {k: v.cpu() for k, v in net.state_dict().items()}} + return ckpt, metrics + + +class DoodleScorer: + """Grades a canvas for a category with the independent classifier (``modal_app.py::train_quickdraw_classifier`` + writes the checkpoint: ``{"categories", "state_dict"}``). ``score`` is P(category); ``passed`` when it is the + classifier's top category. Usable as ``JSPaintEnv(scorer=...)``.""" + + def __init__(self, path: str, device: str = "cpu"): + import torch + + ck = torch.load(path, map_location=device) + self.categories = list(ck["categories"]) + self.net = make_classifier(len(self.categories)) + self.net.load_state_dict(ck["state_dict"]) + self.net.eval().to(device) + self.device = device + + def probabilities(self, pixels: np.ndarray) -> Dict[str, float]: + import torch + + x = torch.from_numpy(canvas_bitmap(pixels))[None, None].to(self.device) + with torch.no_grad(): + p = torch.softmax(self.net(x), -1)[0].cpu().numpy() + return {c: float(v) for c, v in zip(self.categories, p)} + + def __call__(self, pixels: np.ndarray, task: str) -> Dict: + ink = int(ink_mask(pixels).sum()) + if ink == 0: + return {"score": 0.0, "passed": False, "top": None, "ink": 0} + p = self.probabilities(pixels) + top = max(p, key=p.get) + return {"score": round(p.get(task, 0.0), 4), "passed": top == task, "top": top, + "top_p": round(p[top], 4), "ink": ink} + + +__all__ = ["BASE_URL", "TRAIN_CATEGORIES", "HELDOUT_CATEGORIES", "DOODLE_FRAC", "fetch_drawings", "doodle_strokes", + "stroke_length", "DoodleLabeller", "drawn_options", "to_bitmap", "render_strokes", "canvas_bitmap", + "BITMAP", "make_classifier", "classifier_dataset", "train_classifier", "DoodleScorer"] diff --git a/modal_app.py b/modal_app.py index 9c335cfb..6a82338c 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2562,7 +2562,10 @@ def full_eval(model: str, parts: str = ",".join(FULL_EVAL_PARTS), datasets: str def _paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int, model: str, beta: float) -> list: + """Episodes for ``specs``: ``[seed, eps, random_prefix, messy]`` for the circle labeller, or with two more items + ``[..., category, doodle_index]`` for a Quick, Draw! doodle (``laya.quickdraw.DoodleLabeller``).""" from laya.paintdata import play_labelled_episode + from laya.quickdraw import DoodleLabeller, drawn_options, fetch_drawings from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, ModelPolicy os.makedirs(os.path.join(tmp_dir, "images"), exist_ok=True) @@ -2578,13 +2581,29 @@ def save_frame(name, img): agent = VLMAgent(_ckpt_path(model), device="cuda", head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN) driver = ModelPolicy(agent, judge=False) + doodles = {} + for sp in specs: + if len(sp) > 4: + doodles[sp[4]] = max(doodles.get(sp[4], 0), sp[5] + 1) + doodles = {c: fetch_drawings(c, n) for c, n in doodles.items()} + shapes = drawn_options() out, summaries = os.path.join(tmp_dir, "%s-%03d.jsonl" % (split, shard)), [] with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, executable_path=None) as env, \ open(out + ".part", "w") as f: - for seed, eps, prefix, messy in specs: + for sp in specs: + seed, eps, prefix, messy = sp[:4] + task, labeller, env.max_steps = "circle", None, 260 + if len(sp) > 4: + task, doodle = sp[4], doodles[sp[4]][sp[5]] + labeller, env.max_steps = DoodleLabeller(task, doodle, shapes), 400 + env.task = task + if driver is not None: + driver.task = task recs, summ = play_labelled_episode(env, seed, "%s-%d" % (split, seed), save_frame, eps=eps, random_prefix=prefix, judge_every=judge_every, messy=messy, - driver=driver, beta=beta) + driver=driver, beta=beta, task=task, labeller=labeller) + if len(sp) > 4: + summ.update(task=task, doodle=doodle["key_id"]) f.writelines(json.dumps(r) + "\n" for r in recs) summaries.append(summ) os.rename(out + ".part", out) @@ -2611,7 +2630,8 @@ def paint_shard_dagger(tmp_dir: str, split: str, shard: int, specs: list, judge_ @app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, - judge_every: int = 3, model: str = "", beta: float = 0.5): + judge_every: int = 3, model: str = "", beta: float = 0.5, categories: str = "", + per_category: int = 10, val_per_category: int = 2): """Write /data/vqa//{train,val}.jsonl + images/: labelled JSPaint circle episodes. Refuses to replace an existing prepared dataset (they are create-only; ``paint_circle`` was the first, fixed-target labeller). @@ -2626,6 +2646,11 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int With ``model`` (a run under /ckpt), the checkpoint drives each step with probability ``1 - beta`` (DAgger, on L4 containers): the data then covers the states the model itself reaches, labelled with what the labeller would do there. Use a new ``seed`` per round so the episodes differ from the earlier sets'. + + With ``categories`` (comma list of Quick, Draw! categories, or ``train`` for ``laya.quickdraw.TRAIN_CATEGORIES``) + the episodes are doodles instead: ``per_category`` train and ``val_per_category`` val episodes per category, each + following a different human doodle (train uses the category's doodles 0..per_category-1, val the next ones; + the independent classifier trains on doodles from 500 on). ``n_train`` / ``n_val`` are then ignored. """ import random import shutil @@ -2639,10 +2664,22 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int os.makedirs(os.path.join(tmp_dir, "images")) data_vol.commit() rng = random.Random(seed) + from laya.quickdraw import TRAIN_CATEGORIES + + cats = [c.strip() for c in categories.split(",") if c.strip()] + cats = list(TRAIN_CATEGORIES) if cats == ["train"] else cats jobs = [] for split, n, seed0 in (("train", n_train, seed), ("val", n_val, seed + 1_000_000)): specs = [] - for i in range(n): + if cats: + per, first = (per_category, 0) if split == "train" else (val_per_category, per_category) + for ci, c in enumerate(cats): + for d in range(per): + prefix = rng.choice((0, 0, 0, 10, 30)) + specs.append([seed0 + ci * 1000 + d, rng.choice((0.0, 0.05, 0.1, 0.2)), prefix, + bool(prefix) and rng.random() < 1 / 3, c, first + d]) + rng.shuffle(specs) + for i in range(0 if cats else n): prefix = rng.choice((0, 0, 10, 30, 60)) specs.append([seed0 + i, rng.choice((0.0, 0.1, 0.2, 0.3)), prefix, bool(prefix) and rng.random() < 1 / 3]) k = max(1, min(shards, n)) @@ -2655,9 +2692,12 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int split = job[1] summaries[split].extend(res) data_vol.reload() - meta = {"source": "JSPaint %s@%s in headless Chromium, state-based circle labeller (laya/paintdata.py)" - % (JSPAINT_REPO, JSPAINT_COMMIT), "judge_every": judge_every, "driver": model or "labeller", - "beta": beta if model else None} + meta = {"source": "JSPaint %s@%s in headless Chromium, %s" % ( + JSPAINT_REPO, JSPAINT_COMMIT, "Quick, Draw! doodles followed by laya/quickdraw.py DoodleLabeller " + "(quickdraw_dataset simplified ndjson, CC BY 4.0)" if cats else + "state-based circle labeller (laya/paintdata.py)"), + "judge_every": judge_every, "driver": model or "labeller", "beta": beta if model else None, + "categories": cats or None} for split in ("train", "val"): parts = sorted(fn for fn in os.listdir(tmp_dir) if fn.startswith(split + "-") and fn.endswith(".jsonl")) kinds = Counter() @@ -2677,9 +2717,15 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int json.dump(meta, f, indent=2) with open(os.path.join(tmp_dir, "episodes.json"), "w") as f: json.dump(summaries, f) - _write_manifest(tmp_dir, {JSPAINT_REPO: JSPAINT_COMMIT}, + sources = {JSPAINT_REPO: JSPAINT_COMMIT} + if cats: + from laya.quickdraw import BASE_URL + + sources[BASE_URL % ""] = "first recognised doodles per category, CC BY 4.0" + _write_manifest(tmp_dir, sources, dict(n_train=n_train, n_val=n_val, shards=shards, seed=seed, judge_every=judge_every, - model=model, beta=beta)) + model=model, beta=beta, categories=cats, per_category=per_category, + val_per_category=val_per_category)) os.rename(tmp_dir, final_dir) open(os.path.join(final_dir, "_READY"), "w").close() data_vol.commit() @@ -2762,3 +2808,121 @@ def on_step(env, action, pol=pol, pname=pname, steps_log=steps_log): return {k: v for k, v in out.items() if k != "policies"} | { "summary": {p: {k: r[k] for k in ("mean_score", "pass_rate", "mean_steps", "actions")} for p, r in out["policies"].items()}} + + +QUICKDRAW_CLASSIFIER = "/ckpt/quickdraw/classifier.pt" + + +@app.function(image=image, gpu="L4", cpu=8, memory=32768, timeout=2 * 60 * 60, volumes={"/ckpt": ckpt_vol}) +def train_quickdraw_classifier(n_per: int = 3000, epochs: int = 12, seed: int = 0): + """Train the independent doodle scorer (``laya.quickdraw.train_classifier``) on the training and held-out + categories and write it to ``/ckpt/quickdraw/classifier.pt`` (with ``classifier-metrics.json``). Its doodles + are each category's recognised doodles from 500 on, disjoint from the ones the drawing episodes follow. It is + what ``doodle_eval`` grades drawings with, so the drawing model is never graded by its own judgement.""" + import torch + + from laya.quickdraw import HELDOUT_CATEGORIES, TRAIN_CATEGORIES, train_classifier + + cats = list(TRAIN_CATEGORIES + HELDOUT_CATEGORIES) + ckpt, metrics = train_classifier(cats, n_per=n_per, epochs=epochs, device="cuda", seed=seed) + os.makedirs(os.path.dirname(QUICKDRAW_CLASSIFIER), exist_ok=True) + if os.path.exists(QUICKDRAW_CLASSIFIER): + raise SystemExit("%s exists; checkpoints are create-only" % QUICKDRAW_CLASSIFIER) + torch.save(ckpt, QUICKDRAW_CLASSIFIER) + with open(os.path.join(os.path.dirname(QUICKDRAW_CLASSIFIER), "classifier-metrics.json"), "w") as f: + json.dump(metrics, f, indent=2) + ckpt_vol.commit() + print("val accuracy %.3f over %d categories" % (metrics["val_accuracy"], len(cats))) + return metrics + + +@app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=2 * 60 * 60, + volumes={"/cache/hf": hf_vol, "/ckpt": ckpt_vol}) +def doodle_eval_task(model: str, task: str, episodes: int, seed: int, policies: str, first_doodle: int) -> dict: + """``doodle_eval`` for one category: play ``episodes`` with each policy, graded by the classifier.""" + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, model_policy, \ + play_episodes, random_policy + from laya.quickdraw import DoodleLabeller, DoodleScorer, drawn_options, fetch_drawings + + scorer = DoodleScorer(QUICKDRAW_CLASSIFIER, device="cuda") + out = {"task": task, "policies": {}} + agent = None + if "model" in policies: + from laya.vlm import VLMAgent + + agent = VLMAgent(_ckpt_path(model), device="cuda", head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN) + doodles = fetch_drawings(task, first_doodle + episodes)[first_doodle:] + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, task=task, max_steps=400, scorer=scorer, + keep_frames=True) as env: + for pname in policies.split(","): + if pname == "model": + pol = model_policy(agent, task, judge=False) + elif pname == "random": + pol = random_policy(seed) + else: # the labeller following held-back human doodles: what a faithful copier would score + labellers = [DoodleLabeller(task, d, drawn_options()) for d in doodles] + + def pol(env, labellers=labellers): + lab = labellers[env.episode % len(labellers)] + if getattr(lab, "env", None) is not env or lab._episode != env.episode: + lab.reset(env) + lab._episode = env.episode + return lab.action(env, env.visible_pixels()) + res = play_episodes(env, pol, episodes, seed=seed) + out["policies"][pname] = {k: res[k] for k in ("mean_score", "pass_rate", "mean_steps")} | { + "episodes": [{k: e.get(k) for k in ("seed", "score", "passed", "top", "steps", "ended")} + for e in res["results"]]} + if pname == "model" and _ckpt_exists(model): + d = os.path.join(_ckpt_path(model), "doodle_eval_frames") + os.makedirs(d, exist_ok=True) + env.frames[-1].save(os.path.join(d, "%s-last.png" % task.replace(" ", "_"))) + ckpt_vol.commit() + return out + + +def _ckpt_exists(model: str) -> bool: + return os.path.exists(os.path.join(_ckpt_path(model), "vlm_agent_config.json")) + + +@app.function(image=image, timeout=4 * 60 * 60, volumes={"/ckpt": ckpt_vol}) +def doodle_eval(model: str = "paint-circle-v3/best", tasks: str = "train,heldout", episodes: int = 2, + seed: int = 910_000, policies: str = "model,labeller,random", first_doodle: int = 200, + name: str = ""): + """Ask a checkpoint to draw each Quick, Draw! category in JSPaint and grade it with the independent classifier + (``train_quickdraw_classifier``): score = P(requested category), passed = the classifier's top category. + + ``tasks`` takes category names and ``train`` / ``heldout`` (``laya.quickdraw``). Each category runs in its own + L4 container. ``labeller`` follows held-back human doodles (from ``first_doodle`` on, unseen by training and by + the classifier), which is roughly the ceiling; ``random`` is the floor. Results are summarised for the training + and the held-out categories and written to ``/doodle_eval/.json``.""" + from laya.quickdraw import HELDOUT_CATEGORIES, TRAIN_CATEGORIES + + names = [] + for t in tasks.split(","): + t = t.strip() + names += list(TRAIN_CATEGORIES) if t == "train" else list(HELDOUT_CATEGORIES) if t == "heldout" else [t] + jobs = [(model, t, episodes, seed, policies, first_doodle) for t in names] + per_task = {r["task"]: r for r in doodle_eval_task.starmap(jobs)} + summary = {} + for group, cats in (("train", TRAIN_CATEGORIES), ("heldout", HELDOUT_CATEGORIES)): + rows = [per_task[c] for c in cats if c in per_task] + if not rows: + continue + summary[group] = {p: {"mean_score": round(sum(r["policies"][p]["mean_score"] for r in rows) / len(rows), 4), + "pass_rate": round(sum(r["policies"][p]["pass_rate"] for r in rows) / len(rows), 4)} + for p in policies.split(",")} + out = {"model": model, "episodes_per_task": episodes, "seed": seed, "first_doodle": first_doodle, + "summary": summary, "tasks": per_task} + print(json.dumps(summary, indent=2)) + for t in names: + print("%-12s %s" % (t, " ".join("%s %.2f/%.0f%%" % (p, per_task[t]["policies"][p]["mean_score"], + 100 * per_task[t]["policies"][p]["pass_rate"]) + for p in policies.split(",")))) + if _ckpt_exists(model): + ckpt_vol.reload() + d = os.path.join(_ckpt_path(model), "doodle_eval") + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, (name or time.strftime("%Y%m%d-%H%M%S")) + ".json"), "w") as f: + json.dump(out, f, indent=2) + ckpt_vol.commit() + return summary diff --git a/tests/test_quickdraw.py b/tests/test_quickdraw.py new file mode 100644 index 00000000..284b6748 --- /dev/null +++ b/tests/test_quickdraw.py @@ -0,0 +1,84 @@ +"""Quick, Draw! doodles as drawing tasks, offline: stroke placement, resampling, rasterising, and the doodle +labeller's judgement labels (the browser and the network are not needed).""" +import math + +import numpy as np +import pytest + +from laya.quickdraw import (BITMAP, DOODLE_FRAC, DoodleLabeller, _resample, doodle_strokes, drawn_options, + render_strokes, to_bitmap) + +PIL = pytest.importorskip("PIL") +from PIL import Image, ImageDraw # noqa: E402 + +S = 512 +HOUSE = {"key_id": "test", "strokes": [[(0, 120), (0, 255), (200, 255), (200, 120), (0, 120)], + [(0, 120), (100, 0), (200, 120)]]} + + +class FakeEnv: + def __init__(self, cursor=(256.0, 256.0), step=6): + self.canvas_size, self.cursor, self.step_px, self.pen = S, cursor, step, False + from laya.paintenv import compass_moves + + self.moves = compass_moves(32) + + +def test_resample_spacing_and_ends(): + pts = _resample([(0, 0), (30, 0), (30, 12)], 6) + assert pts[0] == (0.0, 0.0) and pts[-1] == (30.0, 12.0) + gaps = [math.dist(a, b) for a, b in zip(pts, pts[1:])] + assert max(gaps) <= 6 + 1e-6 + + +def test_doodle_starts_at_cursor_and_fits(): + strokes = doodle_strokes(HOUSE["strokes"], S, (200.0, 150.0), 6) + assert strokes[0][0] == pytest.approx((200.0, 150.0)) + pts = np.array([p for s in strokes for p in s]) + assert (pts.max(0) - pts.min(0)).max() == pytest.approx(DOODLE_FRAC * S, rel=0.02) + near_edge = doodle_strokes(HOUSE["strokes"], S, (500.0, 500.0), 6) # would run off: shifted to fit + pts = np.array([p for s in near_edge for p in s]) + assert pts.min() >= 8 - 1e-6 and pts.max() <= S - 8 + 1e-6 + + +def test_bitmap_is_cropped_and_normalised(): + bm = to_bitmap(render_strokes(HOUSE["strokes"])) + assert bm.shape == (BITMAP, BITMAP) and bm.max() <= 1.0 and bm.sum() > 20 + assert to_bitmap(np.zeros((64, 64), bool)).sum() == 0 + + +def draw_strokes(strokes, upto=None): + im = Image.new("RGB", (S, S), "white") + d = ImageDraw.Draw(im) + for s in strokes[:upto]: + d.line(s, fill="black", width=4) + return np.asarray(im) + + +def test_labeller_starts_by_pressing_at_the_cursor_and_labels_judgements(): + env = FakeEnv() + lab = DoodleLabeller("house", HOUSE, drawn_options()) + lab.reset(env) + assert lab.action(env, draw_strokes(lab.strokes, 0)) == "PEN_DOWN" + blank = lab.judgements(draw_strokes(lab.strokes, 0)) + assert blank == {"progress": "0", "on_track": "on track", "drawn": "nothing"} + lab.i, lab.j = len(lab.strokes), 0 # as if every stroke were drawn + done = lab.judgements(draw_strokes(lab.strokes)) + assert done == {"progress": "4", "on_track": "on track", "drawn": "house"} + assert lab.action(env, draw_strokes(lab.strokes)) == "DONE" + + +def test_stray_ink_is_off_track(): + env = FakeEnv() + lab = DoodleLabeller("house", HOUSE, drawn_options()) + lab.reset(env) + lab.i = 1 + px = draw_strokes(lab.strokes, 1) + im = Image.fromarray(px.copy()) + ImageDraw.Draw(im).line([(10, 500), (500, 480), (20, 470), (490, 460)], fill="black", width=4) + assert lab.judgements(np.asarray(im))["on_track"] == "off track" + + +def test_drawn_options_cover_training_categories_and_nothing(): + opts = drawn_options(("house", "apple")) + assert list(opts) == ["house", "apple", "nothing"] and opts["apple"].startswith("a doodle of an apple") From f8f5d89ec71ec8479e996b2d8517f99a42e15944 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:04:40 +0000 Subject: [PATCH 18/24] prepare_paint: independent shard calls, retried waits, --resume A lost client connection while the parent waited on 48 doodle shards cancelled all of them. Shards are now spawned individually and each result is awaited with retries on connection errors; --resume keeps the .tmp directory and skips shards whose file is already written. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- modal_app.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/modal_app.py b/modal_app.py index 6a82338c..276debab 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2629,9 +2629,22 @@ def paint_shard_dagger(tmp_dir: str, split: str, shard: int, specs: list, judge_ @app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) +def _get_retrying(call, tries: int = 8): + """``call.get()``, retrying transient client/connection errors (a lost connection while waiting on a long + fan-out cancelled a 48-shard prep once); errors raised by the function itself are not retried.""" + for k in range(tries): + try: + return call.get() + except (modal.exception.ConnectionError, TimeoutError, OSError) as e: + if k == tries - 1: + raise + print("waiting on %s again after %r" % (call.object_id, e), flush=True) + time.sleep(10 * (k + 1)) + + def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, judge_every: int = 3, model: str = "", beta: float = 0.5, categories: str = "", - per_category: int = 10, val_per_category: int = 2): + per_category: int = 10, val_per_category: int = 2, resume: bool = False): """Write /data/vqa//{train,val}.jsonl + images/: labelled JSPaint circle episodes. Refuses to replace an existing prepared dataset (they are create-only; ``paint_circle`` was the first, fixed-target labeller). @@ -2651,6 +2664,10 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int the episodes are doodles instead: ``per_category`` train and ``val_per_category`` val episodes per category, each following a different human doodle (train uses the category's doodles 0..per_category-1, val the next ones; the independent classifier trains on doodles from 500 on). ``n_train`` / ``n_val`` are then ignored. + + Shards run as independent calls; waiting on them retries a lost connection (``_get_retrying``). With + ``resume`` an earlier attempt's ``.tmp`` directory is kept and shards whose file is already there are skipped + (the same arguments give the same shards). """ import random import shutil @@ -2660,8 +2677,9 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int if os.path.exists(final_dir): raise SystemExit("%s exists; prepared datasets are create-only, pass a new --name" % final_dir) tmp_dir = final_dir + ".tmp" - shutil.rmtree(tmp_dir, ignore_errors=True) - os.makedirs(os.path.join(tmp_dir, "images")) + if not resume: + shutil.rmtree(tmp_dir, ignore_errors=True) + os.makedirs(os.path.join(tmp_dir, "images"), exist_ok=True) data_vol.commit() rng = random.Random(seed) from laya.quickdraw import TRAIN_CATEGORIES @@ -2688,7 +2706,12 @@ def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int jobs.append((tmp_dir, split, s, specs[s::k], judge_every) + ((model, beta) if model else ())) summaries = {"train": [], "val": []} shard_fn = paint_shard_dagger if model else paint_shard - for job, res in zip(jobs, list(shard_fn.starmap(jobs))): + done_files = set(os.listdir(tmp_dir)) + todo = [j for j in jobs if "%s-%03d.jsonl" % (j[1], j[2]) not in done_files] + print("%d shards, %d already written" % (len(jobs), len(jobs) - len(todo)), flush=True) + calls = [(job, shard_fn.spawn(*job)) for job in todo] + for job, call in calls: + res = _get_retrying(call) split = job[1] summaries[split].extend(res) data_vol.reload() From fb239a7f97447dea6b8fa9a8dd80104d12aebbf2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:07:34 +0000 Subject: [PATCH 19/24] modal_app: put _get_retrying above prepare_paint's decorator It had been inserted between @app.function and def prepare_paint, so the decorator wrapped the helper and prepare_paint was no longer a Modal function. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- modal_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modal_app.py b/modal_app.py index 276debab..f97d5d3d 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2628,7 +2628,6 @@ def paint_shard_dagger(tmp_dir: str, split: str, shard: int, specs: list, judge_ return _paint_shard(tmp_dir, split, shard, specs, judge_every, model, beta) -@app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) def _get_retrying(call, tries: int = 8): """``call.get()``, retrying transient client/connection errors (a lost connection while waiting on a long fan-out cancelled a 48-shard prep once); errors raised by the function itself are not retried.""" @@ -2642,6 +2641,7 @@ def _get_retrying(call, tries: int = 8): time.sleep(10 * (k + 1)) +@app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, judge_every: int = 3, model: str = "", beta: float = 0.5, categories: str = "", per_category: int = 10, val_per_category: int = 2, resume: bool = False): From 2f0eda32c5d0c9aea7fb07f3b9e2b9ca9f775370 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 08:30:07 +0000 Subject: [PATCH 20/24] ModelPolicy: choose move vs pen action first, then the direction A doodle model toggled PEN_DOWN / PEN_UP for whole episodes: human doodles start in many directions, so the probability of moving spread over the 32 compass options and PEN_UP (0.44) beat every single direction although moving was likelier overall. grouped_choice sums the moves before comparing them with the pen actions (on by default; grouped=False restores the plain argmax). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/paintenv.py | 23 +++++++++++++++++++---- tests/test_quickdraw.py | 10 ++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/laya/paintenv.py b/laya/paintenv.py index a3d0527a..89361c88 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -442,12 +442,25 @@ def judge_canvas(agent, image, task: str = "circle", questions=("drawn", "progre return summarize_judgements(agent.predict({"image": image}, qs, strict=True)["answers"]) +def grouped_choice(probs: Dict[str, float]) -> str: + """Decide in two steps: move or a pen action (``PEN_DOWN`` / ``PEN_UP`` / ``DONE``), with the moves' probability + summed; then, to move, the most likely direction. Taking the single most likely of all 35 options instead + fails when the next direction is uncertain: the moves' probability spreads over many compass points, so a + pen action wins although moving is more likely (a doodle model toggled the pen for whole episodes that way).""" + pen = {a: probs[a] for a in PEN_ACTIONS if a in probs} + moves = {a: p for a, p in probs.items() if a not in PEN_ACTIONS} + if moves and sum(moves.values()) >= max(pen.values(), default=0.0): + return max(moves, key=moves.get) + return max(pen, key=pen.get) + + class ModelPolicy: """The model's most likely action from ``env.state()`` (two frames plus pen state and recent actions), via ``predict``, with the judgement questions (``laya.games.paint_judgements``) asked about the same state in the same call. - ``last`` keeps the latest action answer (probabilities over ``env.actions``, and ``act_probability``, the + The action is chosen with ``grouped_choice`` (move vs pen action first, then the direction) unless + ``grouped=False``. ``last`` keeps the latest action answer (probabilities over ``env.actions``, and ``act_probability``, the checkpoint's act-vs-escalate gate); ``last_judgement`` the latest judgements. With ``judge_stop``, the policy answers ``DONE`` when the pen is up and the model judges the task complete (P(progress = 4) >= ``stop_at``), so its own judgement decides when to stop. @@ -456,8 +469,9 @@ class ModelPolicy: ``strict=True`` and raises rather than silently cut a question.""" def __init__(self, agent, task: str = "circle", judge: bool = True, judge_stop: bool = False, - stop_at: float = 0.5): + stop_at: float = 0.5, grouped: bool = True): self.agent, self.task, self.judge, self.judge_stop, self.stop_at = agent, task, judge, judge_stop, stop_at + self.grouped = grouped self.last, self.last_judgement, self.provenance, self._questions = None, None, None, {} def questions(self, env: JSPaintEnv) -> Dict: @@ -479,7 +493,7 @@ def __call__(self, env: JSPaintEnv) -> str: if (self.judge and self.judge_stop and not env.pen and self.last_judgement["progress_probs"]["4"] >= self.stop_at): return "DONE" - return self.last["choice"] + return grouped_choice(self.last["probabilities"]) if self.grouped else self.last["choice"] def model_policy(agent, task: str = "circle", **kwargs) -> ModelPolicy: @@ -512,4 +526,5 @@ def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step "compass_moves", "TASKS", "TOOLS", "JSPaintServer", "JSPaintEnv", "circle_expert", "random_policy", - "ModelPolicy", "model_policy", "judge_canvas", "summarize_judgements", "play_episodes", "default_chromium"] + "ModelPolicy", "model_policy", "grouped_choice", "judge_canvas", "summarize_judgements", "play_episodes", + "default_chromium"] diff --git a/tests/test_quickdraw.py b/tests/test_quickdraw.py index 284b6748..a7045ed7 100644 --- a/tests/test_quickdraw.py +++ b/tests/test_quickdraw.py @@ -82,3 +82,13 @@ def test_stray_ink_is_off_track(): def test_drawn_options_cover_training_categories_and_nothing(): opts = drawn_options(("house", "apple")) assert list(opts) == ["house", "apple", "nothing"] and opts["apple"].startswith("a doodle of an apple") + + +def test_grouped_choice_moves_when_moving_is_likelier_than_any_pen_action(): + from laya.paintenv import COMPASS, grouped_choice + + probs = {d: 0.02 for d in COMPASS} # moving: 0.66 in total, spread thin + probs.update(E=0.04, PEN_DOWN=0.0, PEN_UP=0.30, DONE=0.02) + assert max(probs, key=probs.get) == "PEN_UP" and grouped_choice(probs) == "E" + probs.update(PEN_UP=0.9) + assert grouped_choice(probs) == "PEN_UP" From 84fed8c564183def206b3ec248dabdc532a549dd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 08:30:22 +0000 Subject: [PATCH 21/24] paintenv: rewrap ModelPolicy docstring Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/paintenv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/laya/paintenv.py b/laya/paintenv.py index 89361c88..32d2759a 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -460,8 +460,8 @@ class ModelPolicy: same call. The action is chosen with ``grouped_choice`` (move vs pen action first, then the direction) unless - ``grouped=False``. ``last`` keeps the latest action answer (probabilities over ``env.actions``, and ``act_probability``, the - checkpoint's act-vs-escalate gate); ``last_judgement`` the latest judgements. With ``judge_stop``, the policy + ``grouped=False``. ``last`` keeps the latest action answer (probabilities over ``env.actions``, and + ``act_probability``, the checkpoint's act-vs-escalate gate); ``last_judgement`` the latest judgements. With ``judge_stop``, the policy answers ``DONE`` when the pen is up and the model judges the task complete (P(progress = 4) >= ``stop_at``), so its own judgement decides when to stop. From e4a2e1e2bef34c0da394d1b0cd61a0194eb1a9f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 08:30:34 +0000 Subject: [PATCH 22/24] paintenv: rewrap ModelPolicy docstring Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/paintenv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/laya/paintenv.py b/laya/paintenv.py index 32d2759a..35526190 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -461,9 +461,9 @@ class ModelPolicy: The action is chosen with ``grouped_choice`` (move vs pen action first, then the direction) unless ``grouped=False``. ``last`` keeps the latest action answer (probabilities over ``env.actions``, and - ``act_probability``, the checkpoint's act-vs-escalate gate); ``last_judgement`` the latest judgements. With ``judge_stop``, the policy - answers ``DONE`` when the pen is up and the model judges the task complete (P(progress = 4) >= ``stop_at``), - so its own judgement decides when to stop. + ``act_probability``, the checkpoint's act-vs-escalate gate); ``last_judgement`` the latest judgements. With + ``judge_stop``, the policy answers ``DONE`` when the pen is up and the model judges the task complete + (P(progress = 4) >= ``stop_at``), so its own judgement decides when to stop. Load the agent with ``head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN``: ``predict`` runs with ``strict=True`` and raises rather than silently cut a question.""" From bf350c13592c9a5d258103314a47b12543824d5d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 08:38:58 +0000 Subject: [PATCH 23/24] ModelPolicy: decide = argmax (default) | grouped | sample grouped choice made the circle checkpoint worse (0.61 -> 0.51 on the same 5 seeds) and did not fix the doodle one, so the default is back to argmax. sample draws move-vs-pen, then the direction, from the model's probabilities: with many plausible doodles per category, sampling commits to one plan where the argmax averages them. doodle_eval takes --decide. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- laya/paintenv.py | 36 +++++++++++++++++++++++++++++------- modal_app.py | 11 ++++++----- tests/test_quickdraw.py | 13 +++++++++++++ 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/laya/paintenv.py b/laya/paintenv.py index 35526190..66ac4975 100644 --- a/laya/paintenv.py +++ b/laya/paintenv.py @@ -454,14 +454,28 @@ def grouped_choice(probs: Dict[str, float]) -> str: return max(pen, key=pen.get) +def sampled_choice(probs: Dict[str, float], rng: random.Random) -> str: + """Sample instead of taking the most likely: move vs each pen action (the moves' probability summed), then the + direction in proportion to its probability. When several plans are plausible (human doodles of one category + start and continue in many ways), sampling commits to one where the argmax averages them into indecision.""" + moves = {a: p for a, p in probs.items() if a not in PEN_ACTIONS} + groups = {"move": sum(moves.values()), **{a: probs[a] for a in PEN_ACTIONS if a in probs}} + pick = rng.choices(list(groups), weights=list(groups.values()))[0] + if pick != "move": + return pick + return rng.choices(list(moves), weights=list(moves.values()))[0] + + class ModelPolicy: """The model's most likely action from ``env.state()`` (two frames plus pen state and recent actions), via ``predict``, with the judgement questions (``laya.games.paint_judgements``) asked about the same state in the same call. - The action is chosen with ``grouped_choice`` (move vs pen action first, then the direction) unless - ``grouped=False``. ``last`` keeps the latest action answer (probabilities over ``env.actions``, and - ``act_probability``, the checkpoint's act-vs-escalate gate); ``last_judgement`` the latest judgements. With + ``decide`` picks the action from the probabilities: ``argmax`` (the most likely of all options, what the + circle checkpoints were trained and evaluated with), ``grouped`` (``grouped_choice``: move vs pen action + first, then the direction) or ``sample`` (``sampled_choice``, seeded by ``seed``). ``last`` keeps the latest + action answer (probabilities over ``env.actions``, and ``act_probability``, the checkpoint's act-vs-escalate + gate); ``last_judgement`` the latest judgements. With ``judge_stop``, the policy answers ``DONE`` when the pen is up and the model judges the task complete (P(progress = 4) >= ``stop_at``), so its own judgement decides when to stop. @@ -469,9 +483,11 @@ class ModelPolicy: ``strict=True`` and raises rather than silently cut a question.""" def __init__(self, agent, task: str = "circle", judge: bool = True, judge_stop: bool = False, - stop_at: float = 0.5, grouped: bool = True): + stop_at: float = 0.5, decide: str = "argmax", seed: int = 0): + if decide not in ("argmax", "grouped", "sample"): + raise ValueError("decide must be argmax, grouped or sample") self.agent, self.task, self.judge, self.judge_stop, self.stop_at = agent, task, judge, judge_stop, stop_at - self.grouped = grouped + self.decide, self.rng = decide, random.Random(seed) self.last, self.last_judgement, self.provenance, self._questions = None, None, None, {} def questions(self, env: JSPaintEnv) -> Dict: @@ -493,7 +509,12 @@ def __call__(self, env: JSPaintEnv) -> str: if (self.judge and self.judge_stop and not env.pen and self.last_judgement["progress_probs"]["4"] >= self.stop_at): return "DONE" - return grouped_choice(self.last["probabilities"]) if self.grouped else self.last["choice"] + probs = self.last["probabilities"] + if self.decide == "grouped": + return grouped_choice(probs) + if self.decide == "sample": + return sampled_choice(probs, self.rng) + return self.last["choice"] def model_policy(agent, task: str = "circle", **kwargs) -> ModelPolicy: @@ -526,5 +547,6 @@ def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step "compass_moves", "TASKS", "TOOLS", "JSPaintServer", "JSPaintEnv", "circle_expert", "random_policy", - "ModelPolicy", "model_policy", "grouped_choice", "judge_canvas", "summarize_judgements", "play_episodes", + "ModelPolicy", "model_policy", "grouped_choice", "sampled_choice", "judge_canvas", "summarize_judgements", + "play_episodes", "default_chromium"] diff --git a/modal_app.py b/modal_app.py index f97d5d3d..7fe3cc86 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2861,7 +2861,8 @@ def train_quickdraw_classifier(n_per: int = 3000, epochs: int = 12, seed: int = @app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=2 * 60 * 60, volumes={"/cache/hf": hf_vol, "/ckpt": ckpt_vol}) -def doodle_eval_task(model: str, task: str, episodes: int, seed: int, policies: str, first_doodle: int) -> dict: +def doodle_eval_task(model: str, task: str, episodes: int, seed: int, policies: str, first_doodle: int, + decide: str = "argmax") -> dict: """``doodle_eval`` for one category: play ``episodes`` with each policy, graded by the classifier.""" from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, model_policy, \ play_episodes, random_policy @@ -2879,7 +2880,7 @@ def doodle_eval_task(model: str, task: str, episodes: int, seed: int, policies: keep_frames=True) as env: for pname in policies.split(","): if pname == "model": - pol = model_policy(agent, task, judge=False) + pol = model_policy(agent, task, judge=False, decide=decide, seed=seed) elif pname == "random": pol = random_policy(seed) else: # the labeller following held-back human doodles: what a faithful copier would score @@ -2910,7 +2911,7 @@ def _ckpt_exists(model: str) -> bool: @app.function(image=image, timeout=4 * 60 * 60, volumes={"/ckpt": ckpt_vol}) def doodle_eval(model: str = "paint-circle-v3/best", tasks: str = "train,heldout", episodes: int = 2, seed: int = 910_000, policies: str = "model,labeller,random", first_doodle: int = 200, - name: str = ""): + name: str = "", decide: str = "argmax"): """Ask a checkpoint to draw each Quick, Draw! category in JSPaint and grade it with the independent classifier (``train_quickdraw_classifier``): score = P(requested category), passed = the classifier's top category. @@ -2924,7 +2925,7 @@ def doodle_eval(model: str = "paint-circle-v3/best", tasks: str = "train,heldout for t in tasks.split(","): t = t.strip() names += list(TRAIN_CATEGORIES) if t == "train" else list(HELDOUT_CATEGORIES) if t == "heldout" else [t] - jobs = [(model, t, episodes, seed, policies, first_doodle) for t in names] + jobs = [(model, t, episodes, seed, policies, first_doodle, decide) for t in names] per_task = {r["task"]: r for r in doodle_eval_task.starmap(jobs)} summary = {} for group, cats in (("train", TRAIN_CATEGORIES), ("heldout", HELDOUT_CATEGORIES)): @@ -2934,7 +2935,7 @@ def doodle_eval(model: str = "paint-circle-v3/best", tasks: str = "train,heldout summary[group] = {p: {"mean_score": round(sum(r["policies"][p]["mean_score"] for r in rows) / len(rows), 4), "pass_rate": round(sum(r["policies"][p]["pass_rate"] for r in rows) / len(rows), 4)} for p in policies.split(",")} - out = {"model": model, "episodes_per_task": episodes, "seed": seed, "first_doodle": first_doodle, + out = {"model": model, "decide": decide, "episodes_per_task": episodes, "seed": seed, "first_doodle": first_doodle, "summary": summary, "tasks": per_task} print(json.dumps(summary, indent=2)) for t in names: diff --git a/tests/test_quickdraw.py b/tests/test_quickdraw.py index a7045ed7..4fa0c829 100644 --- a/tests/test_quickdraw.py +++ b/tests/test_quickdraw.py @@ -92,3 +92,16 @@ def test_grouped_choice_moves_when_moving_is_likelier_than_any_pen_action(): assert max(probs, key=probs.get) == "PEN_UP" and grouped_choice(probs) == "E" probs.update(PEN_UP=0.9) assert grouped_choice(probs) == "PEN_UP" + + +def test_sampled_choice_follows_the_grouped_probabilities(): + import collections + import random + + from laya.paintenv import COMPASS, sampled_choice + + probs = {d: 0.0 for d in COMPASS} + probs.update(E=0.3, N=0.3, PEN_DOWN=0.0, PEN_UP=0.4, DONE=0.0) + rng = random.Random(0) + c = collections.Counter(sampled_choice(probs, rng) for _ in range(4000)) + assert set(c) == {"E", "N", "PEN_UP"} and abs(c["PEN_UP"] / 4000 - 0.4) < 0.03 From b5523fb1b8a26c6d99e879e232876b78757090ed Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 15:57:49 +0000 Subject: [PATCH 24/24] modal_app: prompt_sensitivity, does the drawing policy read the task? Asks the action question under each task prompt for the same val states and reports the mean total-variation distance between the answers, the labelled action's probability under its own prompt vs the others, and how often the top action differs by prompt. A doodle checkpoint drew the same thing for most categories; this measures that directly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01He3SxZcrbzN2YgsDSzN1m4 --- modal_app.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/modal_app.py b/modal_app.py index 7fe3cc86..2411df8e 100644 --- a/modal_app.py +++ b/modal_app.py @@ -2950,3 +2950,66 @@ def doodle_eval(model: str = "paint-circle-v3/best", tasks: str = "train,heldout json.dump(out, f, indent=2) ckpt_vol.commit() return summary + + +@app.function(image=image, gpu="L4", cpu=8, memory=32768, timeout=2 * 60 * 60, + volumes={"/cache/hf": hf_vol, "/data": data_vol.read_only(), "/ckpt": ckpt_vol}) +def prompt_sensitivity(model: str, dataset: str, tasks: str = "circle,square,triangle,line", n: int = 200, + name: str = "", seed: int = 0): + """Does the drawing policy read the task? For ``n`` action records from ``dataset``'s val split (states whose + task is one of ``tasks``), ask the action question once per task prompt, same state, and report: + + * ``tv``: mean total-variation distance between the action distributions under two different prompts (0: the + prompt changes nothing; 1: disjoint); + * ``p_label_true`` / ``p_label_other``: mean probability of the labelled action under the state's own prompt + and under the other prompts. A model that uses the task gives the label more probability with the right one. + Written to ``/prompt_sensitivity/.json`` for a run on the volume.""" + import itertools + import random + + from PIL import Image + + from laya.games import paint_goal, paint_question + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN + from laya.vlm import VLMAgent + + names = [t.strip() for t in tasks.split(",") if t.strip()] + root = "/data/vqa/" + dataset + recs = [] + with open(os.path.join(root, "val.jsonl")) as f: + for line in f: + r = json.loads(line) + if not r["id"].endswith("-action"): + continue + own = [t for t in names if paint_goal(t) in r["question"]["instructions"]] + if own: + recs.append((r, own[0])) + random.Random(seed).shuffle(recs) + recs = recs[:n] + agent = VLMAgent(_ckpt_path(model), device="cuda", head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN) + qs = {t: paint_question(t)["action"] for t in names} + tv, p_true, p_other, rows = [], [], [], [] + for r, own in recs: + state = {"images": [Image.open(os.path.join(root, p)).convert("RGB") for p in r["images"]], + "context": r["state_text"]} + out = agent.predict(state, qs, strict=True)["answers"] + dist = {t: out[t]["probabilities"] for t in names} + label = list(qs[own]["criteria"])[r["label"]] + for a, b in itertools.combinations(names, 2): + tv.append(0.5 * sum(abs(dist[a][k] - dist[b][k]) for k in dist[a])) + p_true.append(dist[own][label]) + p_other.extend(dist[t][label] for t in names if t != own) + rows.append({"id": r["id"], "task": own, "label": label, + "top": {t: max(dist[t], key=dist[t].get) for t in names}}) + out = {"model": model, "dataset": dataset, "tasks": names, "n": len(recs), + "tv": round(sum(tv) / len(tv), 4), "p_label_true": round(sum(p_true) / len(p_true), 4), + "p_label_other": round(sum(p_other) / len(p_other), 4), + "top_differs": round(sum(len(set(x["top"].values())) > 1 for x in rows) / len(rows), 4)} + print(json.dumps(out, indent=2)) + if _ckpt_exists(model): + d = os.path.join(_ckpt_path(model), "prompt_sensitivity") + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, (name or dataset) + ".json"), "w") as f: + json.dump(dict(out, rows=rows), f, indent=2) + ckpt_vol.commit() + return out