diff --git a/docs/paint-circle-metrics.json b/docs/paint-circle-metrics.json new file mode 100644 index 00000000..470d568d --- /dev/null +++ b/docs/paint-circle-metrics.json @@ -0,0 +1,141 @@ +{ + "task": "JSPaint circle, mouse-only, 512x512 canvas, 32 directions x 6 px", + "eval": "modal_app.py::paint_eval, 5 episodes, seeds 900000-900004, verifier laya/circle_verifier.py (pass >= 0.7)", + "runs": { + "paint-circle-v1/best": { + "data": "paint_circle (fixed hidden target, superseded)", + "policies": { + "model": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 260.0, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "expert": { + "mean_score": 0.9669000000000001, + "pass_rate": 1.0, + "mean_steps": 180.8, + "scores": [ + 0.9652, + 0.9718, + 0.9662, + 0.9655, + 0.9658 + ] + }, + "random": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 40.6, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } + } + }, + "paint-circle-v2/best": { + "data": "paint_circle_v2", + "policies": { + "model": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 260.0, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "labeller": { + "mean_score": 0.9355399999999999, + "pass_rate": 1.0, + "mean_steps": 98.2, + "scores": [ + 0.9414, + 0.9365, + 0.9323, + 0.9266, + 0.9409 + ] + }, + "expert": { + "mean_score": 0.9669000000000001, + "pass_rate": 1.0, + "mean_steps": 180.8, + "scores": [ + 0.9652, + 0.9718, + 0.9662, + 0.9655, + 0.9658 + ] + }, + "random": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 40.6, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } + } + }, + "paint-circle-v3/best": { + "data": "paint_circle_v2 + paint_circle_dagger1 (v2 driving, beta 0.5)", + "policies": { + "model": { + "mean_score": 0.6131800000000001, + "pass_rate": 0.4, + "mean_steps": 105.2, + "scores": [ + 0.4635, + 0.4997, + 0.6361, + 0.7043, + 0.7623 + ] + }, + "labeller": { + "mean_score": 0.9355399999999999, + "pass_rate": 1.0, + "mean_steps": 98.2, + "scores": [ + 0.9414, + 0.9365, + 0.9323, + 0.9266, + 0.9409 + ] + }, + "random": { + "mean_score": 0.0, + "pass_rate": 0.0, + "mean_steps": 40.6, + "scores": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } + } + } + } +} \ No newline at end of file diff --git a/examples/jspaint_circle.py b/examples/jspaint_circle.py new file mode 100644 index 00000000..fb90697e --- /dev/null +++ b/examples/jspaint_circle.py @@ -0,0 +1,132 @@ +"""Ask Laya Vision to draw a circle in JSPaint with the mouse only, and score it. + +Each step the model sees two screenshots of the JSPaint canvas (8 steps ago and now, the cursor marked on both) plus +its pen state and last 12 actions as text, and a `choice` question over mouse actions (a 6 px move toward each of 32 +compass directions, pen down, pen up, done) picks the next move, which is sent to the real app as pointer events. +The model is loaded with raised token budgets and asked with strict=True, so the full question is never cut. At the +end the circle verifier scores the true canvas pixels. The scripted expert and a random policy play the same seeds as +reference points, and the model's score is also reported normalized, (model - random) / (expert - random), as in the +games suite. + + pip install -e . playwright pillow + git clone https://github.com/r33drichards/jspaint ../jspaint + python examples/jspaint_circle.py --jspaint ../jspaint --policies expert,random # no model needed + python examples/jspaint_circle.py --jspaint ../jspaint --episodes 5 # + thaitea/laya-vision + python examples/jspaint_circle.py --model checkpoints/my-run --revision --device cuda + +Chromium: set LAYA_CHROMIUM (or --chromium) to an existing binary instead of running `playwright install`. +Each run writes a new directory under --out (create-only): summary.json, and per policy and episode a +trajectory.jsonl (action, cursor, pen, and the model's action probabilities), final.png and episode.gif. +""" +import argparse +import json +import os +import time + +from PIL import Image + +from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, judge_canvas, + model_policy, play_episodes, random_policy) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--jspaint", default=os.path.join(os.path.dirname(__file__), "..", "..", "jspaint"), + help="path to a JSPaint checkout (default: a sibling of this repo)") + ap.add_argument("--policies", default="model,expert,random", help="comma list of model, expert, random") + ap.add_argument("--episodes", type=int, default=5) + ap.add_argument("--seed", type=int, default=900_000) + ap.add_argument("--max-steps", type=int, default=260) + ap.add_argument("--step-px", type=int, default=6) + ap.add_argument("--directions", type=int, default=32, choices=(8, 16, 32)) + ap.add_argument("--model", default="thaitea/laya-vision") + ap.add_argument("--revision", default=None, help="pin the Hub revision of --model") + ap.add_argument("--device", default=None) + ap.add_argument("--chromium", default=None, help="Chromium executable (default: LAYA_CHROMIUM or Playwright's)") + ap.add_argument("--canvas-size", type=int, default=None, + help="square canvas side (default: the model's input image_size, 512 for the released checkpoints)") + ap.add_argument("--no-judge", action="store_true", + help="skip the judgement questions (progress, on track, what is drawn) the model answers each step") + ap.add_argument("--judge-stop", action="store_true", + help="let the model's own judgement end the episode: DONE once it rates the task complete") + ap.add_argument("--headed", action="store_true", help="show the browser window") + ap.add_argument("--out", default="results/jspaint") + args = ap.parse_args() + + out_dir = os.path.join(args.out, time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(out_dir, exist_ok=False) + names = [p.strip() for p in args.policies.split(",") if p.strip()] + summary = {"task": "circle", "jspaint": os.path.abspath(args.jspaint), "args": vars(args), "policies": {}} + + agent = None + if "model" in names: + from laya import load_vlm + + # raised token budgets: the full question and a two-frame state fit with nothing cut + agent = load_vlm(args.model, revision=args.revision, device=args.device, head_max_len=PAINT_HEAD_MAX_LEN, + max_len=PAINT_MAX_LEN) + # A square canvas the size of the model's input, so the screenshot reaches the vision tower unscaled. + canvas_size = args.canvas_size or (agent.prep.image_size if agent is not None else 512) + summary["canvas_size"] = canvas_size + + with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, max_steps=args.max_steps, + step_px=args.step_px, directions=args.directions, + canvas_size=canvas_size, headless=not args.headed, + executable_path=args.chromium, keep_frames=True) as env: + for name in names: + if name == "model": + policy = model_policy(agent, judge=not args.no_judge, judge_stop=args.judge_stop) + elif name == "expert": + policy = circle_expert() + elif name == "random": + policy = random_policy(args.seed) + else: + raise SystemExit("unknown policy %r" % name) + + rows, final_labels = [], {} + + def on_step(env, action): + row = dict(env.trajectory[-1], episode=env.seed) + if name == "model": + row["probabilities"] = policy.last["probabilities"] + row["act_probability"] = policy.last["action"]["act_probability"] + if policy.last_judgement: + row["judgement"] = policy.last_judgement + rows.append(row) + if env.done: + if agent is not None and not args.no_judge: + # what the clean final canvas (no cursor) actually shows, for hindsight relabelling + final_labels[env.seed] = judge_canvas(agent, Image.fromarray(env.canvas_pixels())) + ep_dir = os.path.join(out_dir, name, "seed%d" % env.seed) + os.makedirs(ep_dir) + with open(os.path.join(ep_dir, "trajectory.jsonl"), "w") as f: + f.writelines(json.dumps(r) + "\n" for r in rows) + env.frames[-1].save(os.path.join(ep_dir, "final.png")) + env.frames[0].save(os.path.join(ep_dir, "episode.gif"), save_all=True, + append_images=env.frames[1:], duration=120, loop=0) + rows.clear() + + t = time.time() + res = play_episodes(env, policy, args.episodes, seed=args.seed, on_step=on_step) + res["seconds"] = round(time.time() - t, 1) + for ep in res["results"]: + if ep["seed"] in final_labels: + ep["judged_final"] = final_labels[ep["seed"]] + if name == "model": + res["model"], res["provenance"] = args.model, policy.provenance + summary["policies"][name] = res + print("%-7s mean score %.3f pass rate %.2f mean steps %.1f (%.0fs)" % ( + name, res["mean_score"], res["pass_rate"], res["mean_steps"], res["seconds"])) + + pol = summary["policies"] + if {"model", "expert", "random"} <= set(pol): + e, r = pol["expert"]["mean_score"], pol["random"]["mean_score"] + summary["normalized_model_score"] = (pol["model"]["mean_score"] - r) / (e - r) if e > r else None + print("model normalized score (model - random) / (expert - random): %s" % summary["normalized_model_score"]) + with open(os.path.join(out_dir, "summary.json"), "w") as f: + json.dump(summary, f, indent=2, default=str) + print("wrote %s" % out_dir) + + +if __name__ == "__main__": + main() diff --git a/examples/jspaint_judge_check.py b/examples/jspaint_judge_check.py new file mode 100644 index 00000000..65bbb572 --- /dev/null +++ b/examples/jspaint_judge_check.py @@ -0,0 +1,113 @@ +"""Check whether Laya Vision's judgements track what is really happening on the JSPaint canvas. + +The state-based labeller (``laya.paintdata.LabellerPolicy``) and a random policy play; every --every steps the +model answers the judgement questions (`laya.games.paint_judgements`: progress 0-4, on track or off track, what is +drawn) about the same two-frame state the drawing policy sees. The truth for progress is the share of the 36 +angular sectors of the circle the drawing implies (``laya.paintdata.implied_circle``) that hold ink (0 before +drawing, 1 for a full ring). Reported: + +* how judged progress rises with true progress along the labeller's circles (Spearman rank correlation, and the mean + judged progress per quarter of true progress); +* judged progress and P(on track) for the random policy, which should stay low; +* what the model says the clean final canvas shows, per policy. + + python examples/jspaint_judge_check.py --jspaint ../jspaint --episodes 3 + +Writes a new directory under --out: judgements.jsonl (one row per check) and summary.json. +""" +import argparse +import json +import os +import time + +import numpy as np +from PIL import Image + +from laya.games import paint_judgements +from laya.paintdata import LabellerPolicy, implied_circle, ring_state +from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, judge_canvas, random_policy, + summarize_judgements) + + +def true_progress(env) -> float: + """Share of the sectors of the circle the drawing implies that hold ink (0 with nothing drawn).""" + px = env.visible_pixels() + circle = implied_circle(px, None, env.canvas_size) + return 0.0 if circle is None else ring_state(px, circle)["coverage"] + + +def spearman(a, b) -> float: + ra, rb = np.argsort(np.argsort(a)), np.argsort(np.argsort(b)) + return float(np.corrcoef(ra, rb)[0, 1]) if len(a) > 2 and np.std(ra) and np.std(rb) else float("nan") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--jspaint", default=os.path.join(os.path.dirname(__file__), "..", "..", "jspaint")) + ap.add_argument("--episodes", type=int, default=3) + ap.add_argument("--every", type=int, default=10, help="ask the judgements every N steps") + ap.add_argument("--seed", type=int, default=900_000) + ap.add_argument("--model", default="thaitea/laya-vision") + ap.add_argument("--revision", default=None) + ap.add_argument("--device", default=None) + ap.add_argument("--chromium", default=None) + ap.add_argument("--out", default="results/jspaint-judge") + args = ap.parse_args() + + from laya import load_vlm + + out_dir = os.path.join(args.out, time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(out_dir, exist_ok=False) + agent = load_vlm(args.model, revision=args.revision, device=args.device, head_max_len=PAINT_HEAD_MAX_LEN, + max_len=PAINT_MAX_LEN) + questions = paint_judgements("circle") + rows, finals = [], {} + with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size, + executable_path=args.chromium) as env: + for name, make in (("expert", LabellerPolicy), ("random", lambda: random_policy(args.seed))): + policy = make() + finals[name] = [] + for i in range(args.episodes): + env.reset(args.seed + i) + while not env.done: + if env.steps % args.every == 0: + ans = agent.predict(env.state(), questions, strict=True)["answers"] + rows.append({"policy": name, "seed": args.seed + i, "step": env.steps, "pen": env.pen, + "true_progress": round(true_progress(env), 4), **summarize_judgements(ans)}) + env.step(policy(env)) + final = judge_canvas(agent, Image.fromarray(env.canvas_pixels())) + finals[name].append({"seed": args.seed + i, "verifier": env.result["score"], **final}) + rows.append({"policy": name, "seed": args.seed + i, "step": env.steps, "final": True, + "true_progress": round(true_progress(env), 4), **final}) + print("%-6s seed %d verifier %.2f final canvas judged: %s (progress %.2f, on track %.2f)" % ( + name, args.seed + i, env.result["score"], final["drawn"], final["progress"], final["on_track"])) + + exp = [r for r in rows if r["policy"] == "expert" and not r.get("final")] + rnd = [r for r in rows if r["policy"] == "random"] + tp, jp = [r["true_progress"] for r in exp], [r["progress"] for r in exp] + quarters = {} + for lo, hi in ((0, 0.001), (0.001, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.01)): + sel = [r for r in exp if lo <= r["true_progress"] < hi] + if sel: + quarters["%.2f-%.2f" % (lo, min(hi, 1.0))] = { + "n": len(sel), "judged_progress": round(float(np.mean([r["progress"] for r in sel])), 3), + "p_on_track": round(float(np.mean([r["on_track"] for r in sel])), 3)} + summary = { + "model": args.model, "provenance_revision": getattr(agent, "source", {}).get("revision"), + "every": args.every, "episodes": args.episodes, "seed": args.seed, + "expert_spearman_true_vs_judged_progress": round(spearman(tp, jp), 3), + "expert_by_true_progress": quarters, + "random_mean_judged_progress": round(float(np.mean([r["progress"] for r in rnd])), 3), + "random_mean_p_on_track": round(float(np.mean([r["on_track"] for r in rnd])), 3), + "final_canvas": finals, + } + with open(os.path.join(out_dir, "judgements.jsonl"), "w") as f: + f.writelines(json.dumps(r) + "\n" for r in rows) + with open(os.path.join(out_dir, "summary.json"), "w") as f: + json.dump(summary, f, indent=2) + print(json.dumps({k: v for k, v in summary.items() if k != "final_canvas"}, indent=2)) + print("wrote %s" % out_dir) + + +if __name__ == "__main__": + main() diff --git a/laya/circle_verifier.py b/laya/circle_verifier.py new file mode 100644 index 00000000..9ff1af4a --- /dev/null +++ b/laya/circle_verifier.py @@ -0,0 +1,107 @@ +"""Score how well a canvas shows one drawn circle: the verifier behind the JSPaint "draw a circle" task. + +Pure numpy, so it scores any RGB array (the JSPaint canvas, a PIL image, a synthetic test image) without a browser. +The steps: + +1. **Ink**: pixels that differ from the background colour by more than ``ink_threshold`` in any channel. +2. **Circle fit**: an algebraic least-squares (Kasa) fit of ``x^2 + y^2 + a x + b y + c = 0`` to the ink pixels, giving + the centre ``(cx, cy)`` and radius ``r``. +3. **Components**, each in [0, 1]: + + * ``roundness``: how constant the radius is around the ring, ``1 - std(r_k) / (round_tol * r)``, where ``r_k`` is + the mean distance from the centre of the ring ink in angular sector ``k``. Averaging within a sector cancels the + stroke width, so this measures the shape itself: a clean ring scores ~1, a 16-sided walk ~0.9, an octagon ~0.5, + a square or a 4:3 ellipse 0; + * ``coverage``: the fraction of ``bins`` equal angular sectors around the centre that hold ink on the ring, so an + arc or a C-shape loses the missing sectors; + * ``clean``: the fraction of ink on the ring band (within ``band * r`` of it), so scribbles, fills and stray + strokes count against the drawing; + * ``closure``: 1 when no angular gap is wider than ``max_gap_deg``, falling linearly to 0 at ``max_gap_deg + 90``, + so an unclosed arc scores low even when what was drawn is round; + * ``size``: 1 when ``r`` is between ``min_radius`` and ``max_radius`` of the shorter canvas side and the centre is + on the canvas, else 0 (a dot or a straight line fits a tiny or enormous "circle"). + +4. ``score = size * roundness * coverage * clean * closure``. ``passed`` needs ``score >= pass_score`` and no + angular gap wider than ``max_gap_deg`` (the circle is closed). + +The tolerances pass a circle walked with the environment's 16 fine moves and fail an octagon, lines, arcs, squares, +ellipses, filled discs, dots and scribbles; ``tests/test_circle_verifier.py`` pins both sides. ``rms`` (the RMS +distance of all ink from the fitted circle, over ``r``) is reported alongside for diagnosis. +""" +from typing import Dict, Sequence + +import numpy as np + + +def ink_mask(pixels, bg: Sequence[int] = (255, 255, 255), ink_threshold: int = 60) -> np.ndarray: + """A boolean ``(h, w)`` mask of pixels whose colour differs from ``bg`` by more than ``ink_threshold``.""" + arr = np.asarray(pixels) + if arr.ndim == 2: + arr = np.repeat(arr[..., None], 3, axis=2) + arr = arr[..., :3].astype(np.int16) + return (np.abs(arr - np.asarray(bg, dtype=np.int16)).max(axis=2) > ink_threshold) + + +def fit_circle(xs: np.ndarray, ys: np.ndarray): + """Least-squares (Kasa) circle through the points: ``(cx, cy, r)``.""" + xs, ys = xs.astype(np.float64), ys.astype(np.float64) + a = np.stack([xs, ys, np.ones_like(xs)], axis=1) + b = xs ** 2 + ys ** 2 + (p, q, c), *_ = np.linalg.lstsq(a, b, rcond=None) + cx, cy = p / 2.0, q / 2.0 + return float(cx), float(cy), float(np.sqrt(max(c + cx ** 2 + cy ** 2, 0.0))) + + +def _max_gap_deg(occupied: np.ndarray) -> float: + """Widest run of empty angular bins (wrapping around), in degrees.""" + n = len(occupied) + if not occupied.any(): + return 360.0 + run = best = 0 + for v in np.concatenate([occupied, occupied]): + run = 0 if v else run + 1 + best = max(best, run) + return min(best, n) * 360.0 / n + + +def score_circle(pixels, bg: Sequence[int] = (255, 255, 255), ink_threshold: int = 60, min_ink: int = 30, + round_tol: float = 0.05, band: float = 0.2, bins: int = 36, min_radius: float = 0.08, + max_radius: float = 0.5, max_gap_deg: float = 30.0, pass_score: float = 0.7) -> Dict: + """Score an ``(h, w, 3)`` canvas for one drawn circle. Returns ``score`` in [0, 1], ``passed``, every component, + and the fitted ``cx, cy, r`` (see the module docstring).""" + mask = ink_mask(pixels, bg, ink_threshold) + h, w = mask.shape + ys, xs = np.nonzero(mask) + out = {"score": 0.0, "passed": False, "ink": int(len(xs)), "roundness": 0.0, "rms": None, "coverage": 0.0, + "clean": 0.0, "closure": 0.0, "size": 0.0, "max_gap_deg": 360.0, "cx": None, "cy": None, "r": None} + if len(xs) < min_ink: + return out + cx, cy, r = fit_circle(xs, ys) + out.update(cx=round(cx, 2), cy=round(cy, 2), r=round(r, 2)) + side = min(h, w) + size_ok = min_radius * side <= r <= max_radius * side and 0 <= cx < w and 0 <= cy < h + if r <= 0: + return out + d = np.hypot(xs - cx, ys - cy) + dev = np.abs(d - r) + on_ring = dev <= max(band * r, 3.0) + clean = float(on_ring.mean()) + ang = np.arctan2(ys[on_ring] - cy, xs[on_ring] - cx) + idx = ((ang + np.pi) / (2 * np.pi) * bins).astype(int) % bins + occupied = np.zeros(bins, dtype=bool) + occupied[idx] = True + counts = np.bincount(idx, minlength=bins)[occupied] + radii = np.bincount(idx, weights=d[on_ring], minlength=bins)[occupied] / counts + roundness = float(np.clip(1.0 - np.std(radii) / (round_tol * r), 0.0, 1.0)) + coverage = float(occupied.mean()) + gap = _max_gap_deg(occupied) + closure = float(np.clip(1.0 - (gap - max_gap_deg) / 90.0, 0.0, 1.0)) + score = float(size_ok) * roundness * coverage * clean * closure + rms = float(np.sqrt(np.mean(dev ** 2)) / r) + out.update(score=round(score, 4), roundness=round(roundness, 4), rms=round(rms, 4), coverage=round(coverage, 4), + clean=round(clean, 4), closure=round(closure, 4), size=float(size_ok), max_gap_deg=round(gap, 1), + passed=bool(score >= pass_score and gap <= max_gap_deg)) + return out + + +__all__ = ["ink_mask", "fit_circle", "score_circle"] diff --git a/laya/games.py b/laya/games.py index f3a0b65a..f2037f83 100644 --- a/laya/games.py +++ b/laya/games.py @@ -77,6 +77,89 @@ def snake_question() -> Dict: }} +PAINT_DIRECTIONS = { + "N": "straight up", "NNE": "up and slightly right", "NE": "diagonally up and right", + "ENE": "right and slightly up", "E": "straight right", "ESE": "right and slightly down", + "SE": "diagonally down and right", "SSE": "down and slightly right", "S": "straight down", + "SSW": "down and slightly left", "SW": "diagonally down and left", "WSW": "left and slightly down", + "W": "straight left", "WNW": "left and slightly up", "NW": "diagonally up and left", "NNW": "up and slightly left", +} +PAINT_PEN = { + "PEN_DOWN": "press the mouse button to start drawing", "PEN_UP": "release the mouse button to stop drawing", + "DONE": "the drawing is finished", +} +PAINT_GOALS = {"circle": "draw one round, closed circle, about as big as a third of the canvas height", + "square": "draw one closed square with straight sides, about a third of the canvas across"} + + +def paint_goal(task: str) -> str: + """The task sentence for a drawing task: the hand-written goals for ``circle`` / ``square``, otherwise a simple + doodle of the named thing (any Quick, Draw! category, e.g. ``house`` or ``smiley face``).""" + if task in PAINT_GOALS: + return PAINT_GOALS[task] + article = "an" if task[:1].lower() in "aeiou" else "a" + return "draw %s %s as a simple line doodle, about two fifths of the canvas across" % (article, task) + + +def paint_question(task: str = "circle", directions: int = 32, step_px: int = 6) -> Dict: + """The question for ``laya.paintenv.JSPaintEnv``: a paint canvas, a red cursor, and mouse-only actions (a + ``step_px`` move toward each of ``directions`` compass points, then pen down / pen up / done).""" + from laya.paintenv import compass_bearing, compass_moves + + moves = {m: "move the cursor %d pixels at bearing %g degrees (clockwise from straight up)%s" % ( + step_px, compass_bearing(m), ", " + PAINT_DIRECTIONS[m] if m in PAINT_DIRECTIONS else "") + for m in compass_moves(directions)} + return {"action": { + "type": "choice", + "instructions": "You are using a paint program with only the mouse. You see the canvas a few steps ago " + "and now. The white area is the canvas and the red mark is the mouse cursor: a hollow ring " + "with a cross means the button is up, a filled dot means it is held down and moving draws a " + "black line. Your task: %s. Which mouse action should you take now?" % paint_goal(task), + "criteria": {**moves, **PAINT_PEN}, + }} + + +PAINT_SHAPES = { + "circle": "a round, closed circle", "oval": "an oval or ellipse, round but stretched", + "arc": "a curved line or an unclosed part of a circle", "line": "one or more straight lines", + "square": "a square or rectangle", "triangle": "a triangle", "scribble": "random scribbles", + "nothing": "nothing, the canvas is blank", +} + + +def paint_judgements(task: str = "circle", shapes: Optional[Dict[str, str]] = None) -> Dict: + """The judgement questions asked with ``paint_question`` each step (same state, same ``predict`` call): + + * ``progress`` (``score``, 0-4): how far the canvas is toward the task, the model's own dense signal; + * ``on_track`` (``choice``): whether the drawing so far is heading toward the task. Named options, because on + line drawings the model's yes/no (``noul``) answers were unreliable while named choices were not; + * ``drawn`` (``choice`` over ``PAINT_SHAPES``): what the canvas shows now, whatever the task. At the end of an + episode this labels what was actually drawn, so a failed attempt can be relabelled as an example of that. + ``shapes`` replaces ``PAINT_SHAPES`` as its options (e.g. the Quick, Draw! training categories plus + ``nothing``). + """ + goal = paint_goal(task) + return { + "progress": { + "type": "score", + "instructions": "The task is to %s. Looking at the canvas now, how far along is the drawing?" % goal, + "criteria": ["nothing useful drawn yet", "started, but far from done", "about half done", + "nearly done", "the task is complete"], + }, + "on_track": { + "type": "choice", + "instructions": "The task is to %s. Is the drawing so far heading toward that?" % goal, + "criteria": {"on track": "what is drawn so far could become the task by continuing", + "off track": "what is drawn so far will not become the task"}, + }, + "drawn": { + "type": "choice", + "instructions": "What is drawn on this white canvas?", + "criteria": dict(shapes or PAINT_SHAPES), + }, + } + + CONTROL_GOALS = { "CartPole": "A pole is hinged on a cart; push the cart left or right to keep the pole upright and the cart " "on screen.", diff --git a/laya/paintdata.py b/laya/paintdata.py new file mode 100644 index 00000000..7d69c9a0 --- /dev/null +++ b/laya/paintdata.py @@ -0,0 +1,290 @@ +"""Labelled training data for the JSPaint drawing task: actions and the model's own judgements, from any state. + +The scripted ``laya.paintenv.circle_expert`` only works when it is in charge from the first step. Training data +needs labels for the states a learner actually reaches, including ones after its own mistakes, so this module +labels a state from what is on the canvas: + +* **Target circle, read off the canvas**: the circle is drawn clockwise from its top, so any arc on the canvas + implies its circle: radius ``RADIUS_FRAC`` of the canvas side (about a third of the canvas across) and centre + one radius below the topmost ink (``implied_circle``). With nothing drawn yet, the pen's position is the top. + Nothing about the target is invisible: an earlier labeller used a fixed circle centred on the canvas, and a + model trained on it could not see where to press the button and walked past the start. +* **Coverage**: which of ``SECTORS`` angular sectors around that centre hold ink within ``band`` of the ring, read + from the canvas as displayed (``JSPaintEnv.visible_pixels``, which includes the stroke still being drawn). + **Stray ink**: the share of ink farther than that from the ring. +* **Action** (``label_action``): nothing drawn and the pen up -> press if a circle starting here fits on the + canvas, else move toward the nearest start that fits (away from the edges, which are visible); pen down and on + the ring -> the forward (clockwise) move that stays closest to the radius (``paintenv._around``); pen down but + drifted off the ring, or the ring complete -> ``PEN_UP``; pen up and the ring complete -> ``DONE``; pen up with + a partial ring -> walk to the end of the arc (the start of the uncovered gap, clockwise) and press there. + Stray ink cannot be erased with the mouse-only actions, so the labeller finishes the circle anyway. +* **Judgements** (``label_judgements``), for ``laya.games.paint_judgements``, against the implied circle: + ``progress`` 0-4 from coverage (0 nothing, 1 under 40%, 2 under 75%, 3 short of complete, 4 complete); + ``on_track`` off track once stray ink is more than ``STRAY_OFF_TRACK`` of all ink or the implied circle does + not fit on the canvas; ``drawn`` nothing / arc / circle when the canvas is only ring ink (anything else is left + unlabelled rather than guessed). + +``play_labelled_episode`` runs one episode under a behaviour policy (the labeller with probability ``1 - eps``, +else a random action, optionally after ``random_prefix`` random pen-up moves that put the cursor somewhere else) +and returns training records in the ``laya.vlm_train.load_jsonl_examples`` format. The action label is always the +labeller's action for the state, not the action the behaviour policy took (DAgger-style), with the probability +spread to the two neighbouring compass points (the moves on either side are nearly as good). +""" +import math +import random +from typing import Dict, List, Optional, Tuple + +import numpy as np + +from laya.circle_verifier import ink_mask +from laya.paintenv import COMPASS, PEN_ACTIONS, _around, _toward + +SECTORS = 36 +RADIUS_FRAC = 0.18 +TOP_BAND = 8 # px below the topmost ink used to find the top of the circle +MARGIN = 8 # px a circle keeps from the canvas edge +STRAY_OFF_TRACK = 0.1 +DIRECTION_SPREAD = 0.1 # target probability on each neighbouring compass point of the labelled move + + +def radius(side: int) -> float: + return RADIUS_FRAC * side + + +def target_circle(width: int, height: int, radius_frac: float = 0.3) -> Tuple[float, float, float]: + """A fixed circle centred on the canvas (used by ``examples/jspaint_judge_check.py`` for the old expert).""" + return width / 2.0, height / 2.0, radius_frac * min(width, height) + + +def implied_circle(pixels: np.ndarray, pen_pos=None, side: Optional[int] = None): + """The circle the canvas implies. Its radius is fixed (``radius``) and it is drawn clockwise from its top, so + its top is where the drawing starts: near the topmost ink the ring follows ``y = top + (x - x0)^2 / 2r``, and a + least-squares fit of that parabola (its curvature is known) to the ink within ``TOP_BAND`` px of the topmost + ink gives ``x0`` whichever way the arc runs off. The centre is one radius below. The estimate depends only on + the start of the drawing, so it holds steady while the rest is drawn. With no ink, the circle whose top is + ``pen_pos`` (or None when that is not given).""" + side = side or min(pixels.shape[:2]) + r = radius(side) + ys, xs = np.nonzero(ink_mask(pixels)) + if not len(xs): + if pen_pos is None: + return None + return float(pen_pos[0]), float(pen_pos[1] + r), r + top = ys.min() + band = ys <= top + TOP_BAND + x, y = xs[band].astype(np.float64), ys[band].astype(np.float64) + x0 = float(x.mean()) + if x.max() - x.min() >= 4: + a = 1.0 / (2 * r) + slope, _ = np.polyfit(x, y - a * x * x, 1) # y - a x^2 = -2 a x0 x + c + x0 = float(np.clip(-slope / (2 * a), x.min(), x.max())) + return x0, float(top + 2 + r), r # +2: half the 4 px brush above the path + + +def fits(circle, side: int) -> bool: + cx, cy, r = circle + return MARGIN <= cx - r and cx + r <= side - MARGIN and MARGIN <= cy - r and cy + r <= side - MARGIN + + +def start_point(pos, side: int) -> Tuple[float, float]: + """The nearest cursor position from which a circle drawn clockwise from its top fits on the canvas.""" + r = radius(side) + return (min(max(pos[0], MARGIN + r + 1), side - MARGIN - r - 1), + min(max(pos[1], MARGIN + 1), side - MARGIN - 2 * r - 1)) + + +def ring_state(pixels: np.ndarray, circle, band: Optional[float] = None) -> Dict: + """Coverage of the ring and stray ink, from the canvas pixels.""" + cx, cy, r = circle + band = band if band is not None else max(4.0, 0.08 * r) + ys, xs = np.nonzero(ink_mask(pixels)) + covered = np.zeros(SECTORS, dtype=bool) + if len(xs) == 0: + return {"covered": covered, "coverage": 0.0, "ink": 0, "stray": 0.0} + d = np.hypot(xs - cx, ys - cy) + on = np.abs(d - r) <= band + ang = np.arctan2(ys[on] - cy, xs[on] - cx) + covered[((ang + np.pi) / (2 * np.pi) * SECTORS).astype(int) % SECTORS] = True + return {"covered": covered, "coverage": float(covered.mean()), "ink": int(len(xs)), + "stray": float(1.0 - on.mean())} + + +def _gap_start(covered: np.ndarray, pos, circle) -> Tuple[float, float]: + """The ring point where the nearest uncovered run begins, going clockwise (increasing screen angle).""" + cx, cy, r = circle + starts = [k for k in range(SECTORS) if not covered[k] and covered[k - 1]] or [ + k for k in range(SECTORS) if not covered[k]] + pts = [] + for k in starts: + a = (k + 0.25) / SECTORS * 2 * math.pi - math.pi # a little into the uncovered sector + pts.append((cx + r * math.cos(a), cy + r * math.sin(a))) + return min(pts, key=lambda p: math.dist(p, pos)) + + +def label_action(env, pixels: np.ndarray) -> str: + """The labeller's action for the environment's current state (see the module docstring).""" + side, step, pos = env.canvas_size, env.step_px, env.cursor + circle = implied_circle(pixels, pos if env.pen else None, side) + if circle is None: # nothing drawn, pen up: press where a circle fits, else move to where one does + target = start_point(pos, side) + if math.dist(pos, target) <= 0.75 * step: + return "PEN_DOWN" + return _toward(env.moves, pos, target) + cx, cy, r = circle + ring = ring_state(pixels, circle) + complete = bool(ring["covered"].all()) + off_ring = abs(math.dist(pos, (cx, cy)) - r) + if env.pen: + if complete or off_ring > 1.5 * step: + return "PEN_UP" + return _around(env.moves, pos, (cx, cy), r, step) + if complete: + return "DONE" + target = _gap_start(ring["covered"], pos, circle) + if math.dist(pos, target) <= 0.75 * step and off_ring <= 0.5 * step: + return "PEN_DOWN" + return _toward(env.moves, pos, target) + + +def label_judgements(pixels: np.ndarray) -> Dict[str, Optional[str]]: + """Labels for ``paint_judgements`` against the implied circle: ``progress`` (level index as a string), + ``on_track`` and ``drawn`` (None when the canvas is not a clean ring or part of one).""" + side = min(pixels.shape[:2]) + circle = implied_circle(pixels, None, side) + if circle is None: + return {"progress": "0", "on_track": "on track", "drawn": "nothing"} + ring = ring_state(pixels, circle) + cov, complete = ring["coverage"], bool(ring["covered"].all()) + progress = 4 if complete else 0 if cov == 0 else 1 if cov < 0.4 else 2 if cov < 0.75 else 3 + stray = ring["stray"] > STRAY_OFF_TRACK or not fits(circle, side) + drawn = None if stray else "circle" if complete else "arc" + return {"progress": str(progress), "on_track": "off track" if stray else "on track", "drawn": drawn} + + +def direction_target(options: List[str], action: str) -> List[float]: + """A soft target over the action question's options: the labelled move, with ``DIRECTION_SPREAD`` on each + neighbouring compass point among ``options``; one-hot for pen actions.""" + t = [0.0] * len(options) + idx = options.index(action) + if action in PEN_ACTIONS: + t[idx] = 1.0 + return t + moves = [o for o in options if o not in PEN_ACTIONS] + order = sorted(moves, key=COMPASS.index) + i = order.index(action) + for nb in (order[(i - 1) % len(order)], order[(i + 1) % len(order)]): + t[options.index(nb)] += DIRECTION_SPREAD + t[idx] = 1.0 - sum(t) + return t + + +def _question_record(q: Dict) -> Dict: + return {"type": q["type"], "instructions": q["instructions"], "criteria": q["criteria"]} + + +def play_labelled_episode(env, seed: int, rid: str, save_frame, eps: float = 0.0, random_prefix: int = 0, + judge_every: int = 3, task: str = "circle", messy: bool = False, driver=None, + beta: float = 0.5, labeller=None) -> Tuple[List[Dict], Dict]: + """Play one episode and return (training records, episode summary). + + ``save_frame(name, image) -> path`` stores an observation and returns the path the records should reference. + Each step yields an action record; every ``judge_every`` steps, and once for the final canvas, also one record + per labelled judgement. With ``messy`` the ``random_prefix`` steps may press the button too, leaving stray ink + to judge and recover from; otherwise they are pen-up moves that only relocate the cursor. + + ``driver(env) -> action`` (e.g. a trained ``paintenv.ModelPolicy``) takes the step with probability ``1 - beta`` + instead of the labeller (DAgger): the episode then visits the states the model itself reaches, including its + mistakes, and every one of them is still labelled with the labeller's action. + + ``labeller`` supplies the labels: ``reset(env)`` after the environment resets, ``action(env, pixels)``, + ``judgements(pixels)`` and ``questions(task)`` (the judgement questions). ``CircleLabeller`` by default; + ``laya.quickdraw.DoodleLabeller`` follows a human Quick, Draw! doodle.""" + from laya.games import paint_question + + labeller = labeller or CircleLabeller() + rng = random.Random(seed) + q_act = paint_question(task, env.directions, env.step_px)["action"] + options = list(q_act["criteria"]) + q_judge = labeller.questions(task) + records: List[Dict] = [] + env.reset(seed) + labeller.reset(env) + frame_paths: List[str] = [] + + def frame_path(i: int) -> str: + while len(frame_paths) <= i: + k = len(frame_paths) + frame_paths.append(save_frame("%s-%03d" % (rid, k), env._recent_obs[-1])) + return frame_paths[i] + + def judge_records(base, pixels, t): + for name, lab in labeller.judgements(pixels).items(): + if lab is None: + continue + q = q_judge[name] + idx = int(lab) if q["type"] == "score" else list(q["criteria"]).index(lab) + records.append(dict(base, id="%s-%03d-%s" % (rid, t, name), question=_question_record(q), label=idx)) + + def base_state(t): + now = frame_path(t) + prev = frame_paths[max(0, t - (len(env._recent_obs) - 1))] + return {"images": [prev, now], "state_text": env.state_text()} + + while not env.done: + t = env.steps + pixels = env.visible_pixels() # includes the stroke in progress (canvas_pixels would not) + base = base_state(t) + label = labeller.action(env, pixels) + records.append(dict(base, id="%s-%03d-action" % (rid, t), question=_question_record(q_act), + label=options.index(label), target=direction_target(options, label))) + if t % judge_every == 0: + judge_records(base, pixels, t) + if t < random_prefix: # relocate the cursor; with messy, stray strokes too + act = rng.choice([a for a in env.actions if a != "DONE"] if messy else list(env.moves)) + elif rng.random() < eps: + act = rng.choice([a for a in env.actions if a != "DONE"]) + elif driver is not None and rng.random() >= beta: + act = driver(env) + else: + act = label + env.step(act) + if not env.done: + frame_path(env.steps) + frame_path(env.steps) # the final canvas, judged once more (it is where "complete" and "circle" are true) + judge_records(base_state(env.steps), env.visible_pixels(), env.steps) + summary = {"seed": seed, "eps": eps, "random_prefix": random_prefix, "messy": messy, + "driver": driver is not None, "beta": beta if driver is not None else None, "steps": env.steps, + "records": len(records), "verifier": env.result["score"], + "final": labeller.judgements(env.visible_pixels())} + return records, summary + + +class CircleLabeller: + """The circle labeller (``label_action`` / ``label_judgements``) in the interface ``play_labelled_episode`` + takes.""" + + def reset(self, env) -> None: + pass + + def action(self, env, pixels: np.ndarray) -> str: + return label_action(env, pixels) + + def judgements(self, pixels: np.ndarray) -> Dict[str, Optional[str]]: + return label_judgements(pixels) + + def questions(self, task: str) -> Dict: + from laya.games import paint_judgements + + return paint_judgements(task) + + +class LabellerPolicy: + """``label_action`` as a policy (for checking that the labeller draws a circle on its own).""" + + def __call__(self, env) -> str: + return label_action(env, env.visible_pixels()) + + +__all__ = ["SECTORS", "RADIUS_FRAC", "radius", "implied_circle", "fits", "start_point", "target_circle", + "ring_state", "label_action", "label_judgements", "direction_target", + "play_labelled_episode", "CircleLabeller", "LabellerPolicy"] diff --git a/laya/paintenv.py b/laya/paintenv.py new file mode 100644 index 00000000..66ac4975 --- /dev/null +++ b/laya/paintenv.py @@ -0,0 +1,552 @@ +"""JSPaint as an RL environment: the model sees screenshots of the real app and drives it with the mouse only. + +`JSPaint `_ (an MS Paint clone) runs unmodified in headless Chromium through +Playwright; ``JSPaintServer`` serves a local checkout as static files. Each episode: + +* **canvas** is ``canvas_size`` pixels, square and by default 512 x 512: the size laya-vision's released checkpoints + feed the vision tower (``image_size`` in ``vlm_agent_config.json``). The model then sees the canvas 1:1; a wide + canvas would be squashed to a square first, turning a drawn circle into a tall ellipse. JSPaint reads its canvas + size from ``localStorage`` at start-up, so the size is set there before the page loads (JSPaint is unmodified); +* **reset** clears the canvas to white (``api_for_cypress_tests.reset_for_next_test()``), selects the Brush tool + (so strokes are 4 px wide and visible) and puts the cursor at a seeded point on the canvas; +* **observation** is a screenshot of the canvas with the cursor drawn on it (headless screenshots have no OS cursor): + a red ring when the pen is up, a filled red dot when it is down. ``note()`` carries the pen state and step count + as text; +* **actions** are mouse-only. ``env.actions`` is the discrete set the image model chooses from: a ``step_px`` move + toward each of ``directions`` compass points (32 by default: ``N``, ``NbE``, ``NNE``, ... every 11.25 degrees, 6 px, + fine enough to walk a round circle; 16 and 8 give coarser sets), plus ``PEN_DOWN``, ``PEN_UP`` and + ``DONE``. Each maps onto the tool API ``move_mouse(dx, dy)`` / ``mouse_down()`` / + ``mouse_up()``, which sends real pointer events to the page. ``TOOLS`` describes that API as JSON-schema tools, so + a tool-calling agent can drive the same environment with ``call_tool``; +* **memory**: ``state()`` is what the model sees each step: two images, the canvas ``frame_gap`` steps ago and + now (so the direction of travel is visible), plus the pen state, step count and the last ``history`` actions as + text. One screenshot alone does not say which way the stroke was going or what was just tried; +* **reward** is the task verifier on the true canvas pixels (``canvas_pixels()``, read from the page, so the cursor + overlay never counts as ink): ``laya.circle_verifier.score_circle`` for ``task="circle"``. It is paid at the end of + the episode, or as the per-step change in score with ``reward="shaped"``. An episode ends on ``DONE`` or after + ``max_steps``. + +``circle_expert`` (a scripted policy that walks a circle with the same moves), ``random_policy`` and +``model_policy`` (``VLMAgent.predict`` on the screenshot) are the reference points; ``play_episodes`` runs any of +them. ``examples/jspaint_circle.py`` is the command-line runner. + +Needs ``pip install playwright pillow`` and a Chromium: set ``executable_path`` (or ``LAYA_CHROMIUM``) to use an +existing binary instead of ``playwright install``. +""" +import base64 +import functools +import http.server +import io +import math +import os +import random +import threading +from collections import Counter, deque +from typing import Dict, List, Optional, Tuple + +import numpy as np + +COMPASS = ("N", "NbE", "NNE", "NEbN", "NE", "NEbE", "ENE", "EbN", "E", "EbS", "ESE", "SEbE", "SE", "SEbS", "SSE", + "SbE", "S", "SbW", "SSW", "SWbS", "SW", "SWbW", "WSW", "WbS", "W", "WbN", "WNW", "NWbW", "NW", "NWbN", "NNW", + "NbW") +PEN_ACTIONS = ("PEN_DOWN", "PEN_UP", "DONE") + + +def compass_bearing(name: str) -> float: + """Degrees clockwise from straight up (``N``) of a ``COMPASS`` point.""" + return COMPASS.index(name) * 11.25 + + +def compass_moves(directions: int = 32) -> Dict[str, Tuple[float, float]]: + """``{name: (ux, uy)}`` unit vectors (screen y points down) for 8, 16 or 32 compass points, clockwise from ``N`` + (the 32-point names add the "by" points, e.g. ``NbE`` is one point east of north).""" + if directions not in (8, 16, 32): + raise ValueError("directions must be 8, 16 or 32") + every = 32 // directions + return {COMPASS[i]: (round(math.sin(math.radians(11.25 * i)), 6), round(-math.cos(math.radians(11.25 * i)), 6)) + for i in range(0, 32, every)} + + +MOVES = compass_moves(32) +ACTIONS = tuple(MOVES) + PEN_ACTIONS +TASKS = {"circle": "Draw a circle on the canvas.", "square": "Draw a square on the canvas."} + + +def task_text(task: str) -> str: + """One sentence naming the task; any Quick, Draw! category works (``laya.quickdraw``).""" + return TASKS.get(task) or "Draw %s %s on the canvas." % ("an" if task[:1].lower() in "aeiou" else "a", task) + +TOOLS = [ + {"name": "move_mouse", "description": "Move the mouse by (dx, dy) canvas pixels. If the button is held down, " + "this draws a brush stroke along the way.", + "input_schema": {"type": "object", "properties": {"dx": {"type": "integer"}, "dy": {"type": "integer"}}, + "required": ["dx", "dy"]}}, + {"name": "mouse_down", "description": "Press the left mouse button (start drawing).", + "input_schema": {"type": "object", "properties": {}}}, + {"name": "mouse_up", "description": "Release the left mouse button (stop drawing).", + "input_schema": {"type": "object", "properties": {}}}, + {"name": "screenshot", "description": "Return a screenshot of the canvas with the cursor marked.", + "input_schema": {"type": "object", "properties": {}}}, +] + +CURSOR = (230, 30, 30) +# Token budgets for ``load_vlm`` so the full drawing question (35 options with their descriptions, ~1050 tokens) and +# a two-image state fit without any cutting; the checkpoint defaults (256 / 1024) would truncate every option. +PAINT_HEAD_MAX_LEN = 1536 +PAINT_MAX_LEN = 2560 + + +def default_chromium() -> Optional[str]: + """``LAYA_CHROMIUM``, else the Claude Code sandbox's preinstalled Chromium, else ``None`` (Playwright's own).""" + path = os.environ.get("LAYA_CHROMIUM") or "/opt/pw-browsers/chromium" + return path if os.path.exists(path) else None + + +class JSPaintServer: + """Serve a JSPaint checkout on ``127.0.0.1`` (a free port) from a background thread; ``.url`` is the app.""" + + def __init__(self, root: str): + if not os.path.exists(os.path.join(root, "index.html")): + raise FileNotFoundError("no JSPaint checkout at %r (index.html missing)" % root) + + class Quiet(http.server.SimpleHTTPRequestHandler): + def log_message(self, *args): + pass + + self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), functools.partial(Quiet, directory=root)) + self.url = "http://127.0.0.1:%d/" % self.httpd.server_port + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + + def close(self) -> None: + self.httpd.shutdown() + self.httpd.server_close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + +class JSPaintEnv: + """One headless JSPaint page; ``reset`` starts an episode and ``step(action)`` plays one of ``env.actions``.""" + + def __init__(self, url: str, task: str = "circle", step_px: int = 6, directions: int = 32, max_steps: int = 260, + reward: str = "terminal", headless: bool = True, executable_path: Optional[str] = None, + viewport: Tuple[int, int] = (900, 720), keep_frames: bool = False, canvas_size: int = 512, + frame_gap: int = 8, history: int = 12, scorer=None): + if not task: + raise ValueError("task must name what to draw") + if reward not in ("terminal", "shaped"): + raise ValueError("reward must be 'terminal' or 'shaped'") + from playwright.sync_api import sync_playwright + + self.url, self.task, self.step_px, self.max_steps, self.reward_mode = url, task, step_px, max_steps, reward + self.keep_frames, self.directions = keep_frames, directions + self.frame_gap, self.history = frame_gap, history + self.scorer = scorer # scorer(pixels, task) -> {"score", "passed", ...}; None: the circle verifier + self._recent_obs: deque = deque(maxlen=frame_gap + 1) + self.moves = compass_moves(directions) + self.actions = tuple(self.moves) + PEN_ACTIONS + self._pw = sync_playwright().start() + self.browser = self._pw.chromium.launch(headless=headless, + executable_path=executable_path or default_chromium()) + self.page = self.browser.new_page(viewport={"width": viewport[0], "height": viewport[1]}) + self.canvas_size = canvas_size + self.page.add_init_script("localStorage.setItem('width', '%d'); localStorage.setItem('height', '%d');" + % (canvas_size, canvas_size)) + # Only the app itself: no fonts, analytics or update checks from the network. + self.page.route("**/*", lambda route: route.continue_() if route.request.url.startswith(url) + else route.abort()) + self.page.goto(url) + self.page.wait_for_function("window.api_for_cypress_tests") + self.episode, self.steps, self.pen, self.finished = -1, 0, False, False + self.cursor = (0.0, 0.0) + self.trajectory: List[Dict] = [] + self.frames: List = [] + self._last_score = 0.0 + self.result: Optional[Dict] = None + + # -- episode ---------------------------------------------------------------------------------------------- + def reset(self, seed: int = 0): + """Blank canvas, Brush tool, cursor at a seeded point in the middle half of the canvas. Returns the + observation.""" + self.page.mouse.up() + self.page.evaluate("api_for_cypress_tests.reset_for_next_test()") + self.page.click(".tool[title='Brush']") + r = self.page.evaluate("(() => { const r = document.querySelector('.main-canvas').getBoundingClientRect();" + " return [r.x, r.y, r.width, r.height]; })()") + self.origin, self.width, self.height = (r[0], r[1]), int(r[2]), int(r[3]) + if (self.width, self.height) != (self.canvas_size, self.canvas_size): + raise RuntimeError("JSPaint canvas is %dx%d, expected %dx%d" % (self.width, self.height, self.canvas_size, + self.canvas_size)) + if r[1] + r[3] > self.page.viewport_size["height"] or r[0] + r[2] > self.page.viewport_size["width"]: + raise RuntimeError("the %dx%d canvas does not fit the browser viewport; pass a larger viewport" + % (self.width, self.height)) + rng = random.Random(seed) + self.cursor = (rng.uniform(0.25, 0.75) * self.width, rng.uniform(0.25, 0.75) * self.height) + self.page.mouse.move(*self._client(self.cursor)) + self.episode += 1 + self.seed, self.steps, self.pen, self.finished = seed, 0, False, False + self.trajectory, self.frames, self._last_score, self.result = [], [], 0.0, None + obs = self.render() + self._recent_obs.clear() + self._recent_obs.append(obs) + if self.keep_frames: + self.frames.append(obs) + return obs + + @property + def done(self) -> bool: + return self.finished or self.steps >= self.max_steps + + def step(self, action: str): + """Play one of ``env.actions``. Returns ``(observation, reward, done, info)``; ``info["verifier"]`` holds the + full verifier result once the episode is over.""" + if action not in self.actions: + raise ValueError("unknown action %r" % action) + if self.done: + raise RuntimeError("episode is over; call reset()") + if action in self.moves: + dx, dy = self.moves[action] + self.move_mouse(dx * self.step_px, dy * self.step_px) + elif action == "PEN_DOWN": + self.mouse_down() + elif action == "PEN_UP": + self.mouse_up() + else: + self.finished = True + self.steps += 1 + self.trajectory.append({"step": self.steps, "action": action, "x": round(self.cursor[0], 1), + "y": round(self.cursor[1], 1), "pen": self.pen}) + info: Dict = {} + reward = 0.0 + if self.done: + self.mouse_up() + self.result = self.verify() + info["verifier"] = self.result + reward = self.result["score"] - (self._last_score if self.reward_mode == "shaped" else 0.0) + elif self.reward_mode == "shaped": + score = self.verify()["score"] + reward, self._last_score = score - self._last_score, score + obs = self.render() + self._recent_obs.append(obs) + if self.keep_frames: + self.frames.append(obs) + return obs, reward, self.done, info + + def verify(self) -> Dict: + """The episode's grade: ``scorer`` if one was given (e.g. ``laya.quickdraw.DoodleScorer`` for any + category), else the circle verifier.""" + from laya.circle_verifier import score_circle + + if self.scorer is not None: + return self.scorer(self.canvas_pixels(), self.task) + return score_circle(self.canvas_pixels()) + + # -- mouse-only tool API ------------------------------------------------------------------------------------ + def _client(self, xy) -> Tuple[float, float]: + return self.origin[0] + xy[0], self.origin[1] + xy[1] + + def move_mouse(self, dx: float, dy: float) -> None: + """Move by ``(dx, dy)`` canvas pixels, clamped to the canvas; draws while the button is down.""" + x = min(max(self.cursor[0] + dx, 1.0), self.width - 2.0) + y = min(max(self.cursor[1] + dy, 1.0), self.height - 2.0) + self.cursor = (x, y) + steps = max(2, int(math.hypot(dx, dy) // 4)) if self.pen else 1 + self.page.mouse.move(*self._client(self.cursor), steps=steps) + + def mouse_down(self) -> None: + if not self.pen: + self.page.mouse.down() + self.pen = True + + def mouse_up(self) -> None: + if self.pen: + self.page.mouse.up() + self.pen = False + + def call_tool(self, name: str, args: Optional[Dict] = None): + """Run one of ``TOOLS`` by name (for tool-calling agents); ``screenshot`` returns the observation.""" + args = args or {} + if name == "move_mouse": + self.move_mouse(float(args["dx"]), float(args["dy"])) + elif name == "mouse_down": + self.mouse_down() + elif name == "mouse_up": + self.mouse_up() + elif name == "screenshot": + return self.render() + else: + raise ValueError("unknown tool %r" % name) + return None + + # -- observations -------------------------------------------------------------------------------------------- + def screenshot(self, full_page: bool = False): + """A PIL screenshot of the canvas (or the whole app with ``full_page``), without the cursor.""" + from PIL import Image + + clip = None if full_page else {"x": self.origin[0], "y": self.origin[1], "width": self.width, + "height": self.height} + return Image.open(io.BytesIO(self.page.screenshot(clip=clip))).convert("RGB") + + def render(self, full_page: bool = False): + """The observation: a screenshot with the cursor drawn on it (ring = pen up, dot = pen down).""" + from PIL import ImageDraw + + img = self.screenshot(full_page) + x, y = self._client(self.cursor) if full_page else self.cursor + d = ImageDraw.Draw(img) + if self.pen: + d.ellipse([x - 5, y - 5, x + 5, y + 5], fill=CURSOR) + else: + d.ellipse([x - 7, y - 7, x + 7, y + 7], outline=CURSOR, width=2) + d.line([x - 11, y, x + 11, y], fill=CURSOR) + d.line([x, y - 11, x, y + 11], fill=CURSOR) + return img + + def state_text(self) -> str: + """The text half of ``state()``: which image is which, the pen state, the step count and recent actions. + Training records store exactly this string (``laya.paintdata``), so play and training see the same input.""" + recent = [t["action"] for t in self.trajectory[-self.history:]] + return ("The first image is the canvas %d steps ago, the second is now. The pen is %s. Step %d of %d. " + "Recent actions, oldest first: %s." % ( + len(self._recent_obs) - 1, "down (drawing)" if self.pen else "up (not drawing)", + self.steps + 1, self.max_steps, " ".join(recent) if recent else "none yet")) + + def state(self) -> Dict: + """What the model sees this step: ``images`` = [canvas ``frame_gap`` steps ago (the first frame early in an + episode), canvas now], both with the cursor drawn on, and ``context`` = ``state_text()``. The keys match a + training record's (``images`` + ``state_text``, which the loader passes as ``context``).""" + return {"images": [self._recent_obs[0], self._recent_obs[-1]], "context": self.state_text()} + + def note(self) -> str: + return "Task: %s The pen is %s. Step %d of %d." % (task_text(self.task), "down (drawing)" if self.pen else + "up (not drawing)", self.steps + 1, self.max_steps) + + def canvas_pixels(self) -> np.ndarray: + """The true canvas as an ``(h, w, 3)`` uint8 array, read from the page (transparent pixels read as white).""" + from PIL import Image + + data = self.page.evaluate("main_canvas.toDataURL('image/png')") + img = Image.open(io.BytesIO(base64.b64decode(data.split(",", 1)[1]))).convert("RGBA") + white = Image.new("RGBA", img.size, (255, 255, 255, 255)) + return np.asarray(Image.alpha_composite(white, img).convert("RGB")) + + def visible_pixels(self, hide_cursor_px: int = 5) -> np.ndarray: + """The canvas as it looks on screen, as an ``(h, w, 3)`` uint8 array, without our cursor overlay. Unlike + ``canvas_pixels`` this includes a stroke still being drawn: JSPaint keeps the stroke on an overlay and only + commits it to the canvas when the button is released. JSPaint also previews the brush tip under the mouse + pointer, which is not ink, so pixels within ``hide_cursor_px`` of the cursor are painted white.""" + arr = np.array(self.screenshot()) + if hide_cursor_px: + yy, xx = np.ogrid[: arr.shape[0], : arr.shape[1]] + arr[(xx - self.cursor[0]) ** 2 + (yy - self.cursor[1]) ** 2 <= hide_cursor_px ** 2] = 255 + return arr + + def close(self) -> None: + self.browser.close() + self._pw.stop() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + +# -- policies ------------------------------------------------------------------------------------------------------ +def _toward(moves, pos, target) -> str: + """The move whose unit direction best matches ``target - pos``.""" + vx, vy = target[0] - pos[0], target[1] - pos[1] + return max(moves, key=lambda a: moves[a][0] * vx + moves[a][1] * vy) + + +def _around(moves, pos, centre, r, step) -> str: + """Of the moves that advance clockwise round ``centre`` by at least a third of a step, the one that lands + closest to radius ``r`` (ties to the larger advance).""" + ang = math.atan2(pos[1] - centre[1], pos[0] - centre[0]) + + def outcome(a): + x, y = pos[0] + moves[a][0] * step, pos[1] + moves[a][1] * step + adv = (math.atan2(y - centre[1], x - centre[0]) - ang + math.pi) % (2 * math.pi) - math.pi + return abs(math.hypot(x - centre[0], y - centre[1]) - r), -adv + + forward = [a for a in moves if -outcome(a)[1] >= step / r / 3] or list(moves) + return min(forward, key=outcome) + + +def circle_expert(radius_frac: float = 0.3): + """A scripted policy: walk (pen up) straight to the nearest point of a circle centred on the canvas, press once + within half a step of its radius, trace one + full turn plus a little overlap (each step takes the forward move that stays closest to the radius), release, + ``DONE``. Deterministic given the environment state.""" + state: Dict = {} + + def policy(env: JSPaintEnv) -> str: + if state.get("episode") != (id(env), env.episode): + cx, cy = env.width / 2.0, env.height / 2.0 + r = radius_frac * min(env.width, env.height) + state.clear() + state.update(episode=(id(env), env.episode), c=(cx, cy), r=r, phase="approach", turned=0.0, prev=None) + cx, cy = state["c"] + r, pos, step = state["r"], env.cursor, env.step_px + if state["phase"] == "approach": + d = math.dist(pos, (cx, cy)) + if abs(d - r) > step * 0.5: + ux, uy = ((pos[0] - cx) / d, (pos[1] - cy) / d) if d > 1e-6 else (1.0, 0.0) + return _toward(env.moves, pos, (cx + r * ux, cy + r * uy)) + state["phase"] = "trace" + state["prev"] = math.atan2(pos[1] - cy, pos[0] - cx) + return "PEN_DOWN" + if state["phase"] == "trace": + ang = math.atan2(pos[1] - cy, pos[0] - cx) + delta = (ang - state["prev"] + math.pi) % (2 * math.pi) - math.pi + state["turned"] += delta + state["prev"] = ang + if state["turned"] < 2 * math.pi + 0.3: + return _around(env.moves, pos, (cx, cy), r, step) + state["phase"] = "lift" + return "PEN_UP" + return "DONE" + + return policy + + +def random_policy(seed: int = 0): + rng = random.Random(seed) + return lambda env: rng.choice(env.actions) + + +def summarize_judgements(answers: Dict) -> Dict: + """The judgement answers from ``predict`` as a compact record: expected progress (0-4) and its distribution, + P(on track), and the top ``drawn`` label with its distribution.""" + out = {} + if "progress" in answers: + out["progress"] = answers["progress"]["score"] + out["progress_probs"] = answers["progress"]["probabilities"] + if "on_track" in answers: + out["on_track"] = answers["on_track"]["probabilities"]["on track"] + if "drawn" in answers: + out["drawn"] = answers["drawn"]["choice"] + out["drawn_probs"] = answers["drawn"]["probabilities"] + return out + + +def judge_canvas(agent, image, task: str = "circle", questions=("drawn", "progress", "on_track")) -> Dict: + """Ask the judgement questions about one image (e.g. the clean final canvas, ``Image.fromarray(env. + canvas_pixels())``, with no cursor on it). Used at the end of an episode to label what was actually drawn.""" + from laya.games import paint_judgements + + qs = {k: v for k, v in paint_judgements(task).items() if k in questions} + return summarize_judgements(agent.predict({"image": image}, qs, strict=True)["answers"]) + + +def grouped_choice(probs: Dict[str, float]) -> str: + """Decide in two steps: move or a pen action (``PEN_DOWN`` / ``PEN_UP`` / ``DONE``), with the moves' probability + summed; then, to move, the most likely direction. Taking the single most likely of all 35 options instead + fails when the next direction is uncertain: the moves' probability spreads over many compass points, so a + pen action wins although moving is more likely (a doodle model toggled the pen for whole episodes that way).""" + pen = {a: probs[a] for a in PEN_ACTIONS if a in probs} + moves = {a: p for a, p in probs.items() if a not in PEN_ACTIONS} + if moves and sum(moves.values()) >= max(pen.values(), default=0.0): + return max(moves, key=moves.get) + return max(pen, key=pen.get) + + +def sampled_choice(probs: Dict[str, float], rng: random.Random) -> str: + """Sample instead of taking the most likely: move vs each pen action (the moves' probability summed), then the + direction in proportion to its probability. When several plans are plausible (human doodles of one category + start and continue in many ways), sampling commits to one where the argmax averages them into indecision.""" + moves = {a: p for a, p in probs.items() if a not in PEN_ACTIONS} + groups = {"move": sum(moves.values()), **{a: probs[a] for a in PEN_ACTIONS if a in probs}} + pick = rng.choices(list(groups), weights=list(groups.values()))[0] + if pick != "move": + return pick + return rng.choices(list(moves), weights=list(moves.values()))[0] + + +class ModelPolicy: + """The model's most likely action from ``env.state()`` (two frames plus pen state and recent actions), via + ``predict``, with the judgement questions (``laya.games.paint_judgements``) asked about the same state in the + same call. + + ``decide`` picks the action from the probabilities: ``argmax`` (the most likely of all options, what the + circle checkpoints were trained and evaluated with), ``grouped`` (``grouped_choice``: move vs pen action + first, then the direction) or ``sample`` (``sampled_choice``, seeded by ``seed``). ``last`` keeps the latest + action answer (probabilities over ``env.actions``, and ``act_probability``, the checkpoint's act-vs-escalate + gate); ``last_judgement`` the latest judgements. With + ``judge_stop``, the policy answers ``DONE`` when the pen is up and the model judges the task complete + (P(progress = 4) >= ``stop_at``), so its own judgement decides when to stop. + + Load the agent with ``head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN``: ``predict`` runs with + ``strict=True`` and raises rather than silently cut a question.""" + + def __init__(self, agent, task: str = "circle", judge: bool = True, judge_stop: bool = False, + stop_at: float = 0.5, decide: str = "argmax", seed: int = 0): + if decide not in ("argmax", "grouped", "sample"): + raise ValueError("decide must be argmax, grouped or sample") + self.agent, self.task, self.judge, self.judge_stop, self.stop_at = agent, task, judge, judge_stop, stop_at + self.decide, self.rng = decide, random.Random(seed) + self.last, self.last_judgement, self.provenance, self._questions = None, None, None, {} + + def questions(self, env: JSPaintEnv) -> Dict: + from laya.games import paint_judgements, paint_question + + key = (self.task, env.directions, env.step_px) + if key not in self._questions: + q = paint_question(self.task, env.directions, env.step_px) + if self.judge: + q.update(paint_judgements(self.task)) + self._questions[key] = q + return self._questions[key] + + def __call__(self, env: JSPaintEnv) -> str: + out = self.agent.predict(env.state(), self.questions(env), strict=True) + answers = out["answers"] + self.last, self.provenance = answers["action"], out.get("provenance") + self.last_judgement = summarize_judgements(answers) if self.judge else None + if (self.judge and self.judge_stop and not env.pen + and self.last_judgement["progress_probs"]["4"] >= self.stop_at): + return "DONE" + probs = self.last["probabilities"] + if self.decide == "grouped": + return grouped_choice(probs) + if self.decide == "sample": + return sampled_choice(probs, self.rng) + return self.last["choice"] + + +def model_policy(agent, task: str = "circle", **kwargs) -> ModelPolicy: + return ModelPolicy(agent, task, **kwargs) + + +def play_episodes(env: JSPaintEnv, policy, episodes: int, seed: int = 0, on_step=None) -> Dict: + """Play ``episodes`` seeded episodes (seed ``seed + i``) with ``policy(env) -> action``. ``on_step(env, action)`` + is called after each step (for logging). Returns per-episode verifier results and the summary.""" + counts, eps = Counter(), [] + for i in range(episodes): + env.reset(seed + i) + while not env.done: + a = policy(env) + counts[a] += 1 + env.step(a) + if on_step: + on_step(env, a) + eps.append({"seed": seed + i, "steps": env.steps, "ended": "done" if env.finished else "capped", + **env.result}) + scores = [e["score"] for e in eps] + return {"task": env.task, "episodes": episodes, "seed": seed, "directions": env.directions, + "step_px": env.step_px, "max_steps": env.max_steps, + "mean_score": float(np.mean(scores)), "median_score": float(np.median(scores)), + "pass_rate": float(np.mean([e["passed"] for e in eps])), + "mean_steps": float(np.mean([e["steps"] for e in eps])), "actions": dict(counts), "results": eps} + + +__all__ = ["PAINT_HEAD_MAX_LEN", "PAINT_MAX_LEN", "ACTIONS", "MOVES", "COMPASS", "PEN_ACTIONS", "compass_bearing", + "compass_moves", "TASKS", "TOOLS", + "JSPaintServer", + "JSPaintEnv", "circle_expert", "random_policy", + "ModelPolicy", "model_policy", "grouped_choice", "sampled_choice", "judge_canvas", "summarize_judgements", + "play_episodes", + "default_chromium"] diff --git a/laya/quickdraw.py b/laya/quickdraw.py new file mode 100644 index 00000000..7a09f00f --- /dev/null +++ b/laya/quickdraw.py @@ -0,0 +1,353 @@ +"""Human doodles from Google's Quick, Draw! as drawing tasks for the JSPaint environment. + +`Quick, Draw! `_ has about 50 million doodles in 345 +categories, each stored as the pen strokes a person drew, in order (CC BY 4.0). This module turns them into tasks +for ``laya.paintenv.JSPaintEnv`` ("draw a house"): + +* ``fetch_drawings`` reads the first recognised doodles of a category from the simplified ndjson files (coordinates + 0-255, strokes simplified with Ramer-Douglas-Peucker), with an HTTP range request so a category costs a few + megabytes, not the whole file. +* ``doodle_strokes`` scales a doodle to ``DOODLE_FRAC`` of the canvas, starts it where the cursor is (shifted only + as far as needed to fit on the canvas, which is visible), and resamples every stroke to the environment's step. +* ``DoodleLabeller`` follows that doodle from any state: walk to the start of the next stroke, press, trace it with + the compass moves, release, and ``DONE`` after the last stroke. If the pen strays, it lifts and returns to where + the stroke left off, so it can label the states a model reaches after its own mistakes (DAgger). Its judgement + labels come from the requested doodle: progress is the share of the doodle's length drawn, on track means little + ink lies away from the doodle's strokes, and ``drawn`` is the category once the doodle is complete. +* ``render_strokes`` / ``to_bitmap`` rasterise doodles and canvases the same way for the independent scorer + (``modal_app.py::train_quickdraw_classifier``), which grades a drawing by the probability a separately trained + classifier gives the requested category, so the model is never graded by its own judgement. + +The episode's doodle is not shown to the model: it only gets the category name, as a person would. +""" +import json +import math +import urllib.parse +import urllib.request +from typing import Dict, List, Optional, Sequence, Tuple + +import numpy as np + +from laya.circle_verifier import ink_mask +from laya.paintenv import _toward + +BASE_URL = "https://storage.googleapis.com/quickdraw_dataset/full/simplified/%s.ndjson" +TRAIN_CATEGORIES = ( + "circle", "square", "triangle", "line", "zigzag", "star", "hexagon", "octagon", "diamond", "moon", "sun", + "cloud", "house", "tree", "fish", "cup", "eye", "envelope", "umbrella", "key", "apple", "mountain", + "smiley face", "hourglass", +) +HELDOUT_CATEGORIES = ("ladder", "stop sign", "cat", "flower", "lightning", "hat") +DOODLE_FRAC = 0.45 # the doodle's longer side, as a share of the canvas side +MAX_STROKES = 8 +MARGIN = 8 + + +def fetch_drawings(category: str, n: int, max_bytes: int = 4_000_000, skip: int = 0, + max_strokes: int = MAX_STROKES) -> List[Dict]: + """The first ``n`` recognised doodles of ``category`` with at most ``max_strokes`` strokes, after skipping + ``skip`` of them (so disjoint sets can be drawn from the same file). Each is ``{"key_id", "strokes": [[(x, y), + ...], ...]}`` in the dataset's 0-255 coordinates.""" + url = BASE_URL % urllib.parse.quote(category) + req = urllib.request.Request(url, headers={"Range": "bytes=0-%d" % (max_bytes - 1)}) + with urllib.request.urlopen(req, timeout=120) as r: + text = r.read().decode("utf-8", errors="ignore") + out, seen = [], 0 + for line in text.split("\n")[:-1]: # the last line may be cut by the range + d = json.loads(line) + if not d.get("recognized") or len(d["drawing"]) > max_strokes: + continue + seen += 1 + if seen <= skip: + continue + out.append({"key_id": d["key_id"], "strokes": [list(zip(xs, ys)) for xs, ys in d["drawing"]]}) + if len(out) >= n: + break + return out + + +def _resample(points: Sequence[Tuple[float, float]], spacing: float) -> List[Tuple[float, float]]: + """Points every ``spacing`` px along a polyline (always keeping its ends).""" + pts = [tuple(map(float, p)) for p in points] + if len(pts) == 1: + return pts + out, carry = [pts[0]], 0.0 + for a, b in zip(pts, pts[1:]): + seg = math.dist(a, b) + d = spacing - carry + while d <= seg: + t = d / seg + out.append((a[0] + t * (b[0] - a[0]), a[1] + t * (b[1] - a[1]))) + d += spacing + carry = seg - (d - spacing) + if math.dist(out[-1], pts[-1]) > 1e-6: + out.append(pts[-1]) + return out + + +def doodle_strokes(strokes: Sequence[Sequence[Tuple[float, float]]], side: int, start: Tuple[float, float], + step: float) -> List[List[Tuple[float, float]]]: + """Canvas-space strokes: the doodle scaled so its longer side is ``DOODLE_FRAC * side``, placed so its first + point is at ``start`` (shifted as little as needed to keep it ``MARGIN`` px inside the canvas), and resampled + every ``step`` px.""" + pts = np.array([p for s in strokes for p in s], dtype=np.float64) + lo, hi = pts.min(0), pts.max(0) + scale = DOODLE_FRAC * side / max(1.0, float((hi - lo).max())) + first = np.array(strokes[0][0], dtype=np.float64) + off = np.array(start, dtype=np.float64) - first * scale + blo, bhi = lo * scale + off, hi * scale + off + off += np.maximum(0.0, MARGIN - blo) - np.maximum(0.0, bhi - (side - MARGIN)) + return [_resample([tuple(np.array(p) * scale + off) for p in s], step) for s in strokes] + + +def stroke_length(stroke: Sequence[Tuple[float, float]]) -> float: + return sum(math.dist(a, b) for a, b in zip(stroke, stroke[1:])) + + +class DoodleLabeller: + """Follows one human doodle (``play_labelled_episode``'s labeller interface). ``drawing`` is a + ``fetch_drawings`` item; it is placed on the canvas at ``reset`` from where the cursor starts.""" + + def __init__(self, category: str, drawing: Dict, shapes: Optional[Dict[str, str]] = None, + band: float = 10.0, stray_off_track: float = 0.15): + self.category, self.drawing, self.band, self.stray_off_track = category, drawing, band, stray_off_track + self.shapes = shapes + + def reset(self, env) -> None: + self.env = env + self.strokes = doodle_strokes(self.drawing["strokes"], env.canvas_size, env.cursor, env.step_px) + self.i, self.j = 0, 0 # next stroke, next point in it + self.lengths = [stroke_length(s) for s in self.strokes] + self.total = max(1e-6, sum(self.lengths)) + self._target_mask = None + + # -- progress along the doodle ----------------------------------------------------------------------------- + def _advance(self, env) -> None: + """Move the pointer past points the pen (down) has reached; start the next stroke once a finished one is + released.""" + if self.i >= len(self.strokes): + return + s, step = self.strokes[self.i], env.step_px + if env.pen: + window = range(self.j, min(len(s), self.j + 4)) + reached = [k for k in window if math.dist(env.cursor, s[k]) <= 0.8 * step] + if reached: + self.j = reached[-1] + 1 + elif self.j >= len(s): + self.i, self.j = self.i + 1, 0 + + def drawn_fraction(self) -> float: + done = sum(self.lengths[: self.i]) + if self.i < len(self.strokes) and self.j > 0: + s = self.strokes[self.i] + done += stroke_length(s[: min(self.j, len(s))]) + return min(1.0, done / self.total) + + @property + def complete(self) -> bool: + return self.i >= len(self.strokes) + + # -- labeller interface ------------------------------------------------------------------------------------ + def action(self, env, pixels: np.ndarray) -> str: + self._advance(env) + step, pos = env.step_px, env.cursor + if self.complete: + return "PEN_UP" if env.pen else "DONE" + s = self.strokes[self.i] + if env.pen: + if self.j >= len(s): + return "PEN_UP" + if math.dist(pos, s[self.j]) > 3 * step: # strayed: lift, then come back to where the stroke left off + return "PEN_UP" + return _toward(env.moves, pos, s[self.j]) + target = s[min(max(self.j - 1, 0), len(s) - 1)] # the stroke's start, or where it was left off + if math.dist(pos, target) <= 0.75 * step: + return "PEN_DOWN" + return _toward(env.moves, pos, target) + + def _stray(self, pixels: np.ndarray) -> float: + """Share of the ink farther than ``band`` px from every stroke of the doodle.""" + ink = ink_mask(pixels) + if not ink.any(): + return 0.0 + if self._target_mask is None or self._target_mask.shape != ink.shape: + from PIL import Image, ImageDraw + + im = Image.new("L", (ink.shape[1], ink.shape[0]), 0) + d = ImageDraw.Draw(im) + for s in self.strokes: + if len(s) > 1: + d.line(s, fill=255, width=int(2 * self.band)) + d.ellipse([s[0][0] - self.band, s[0][1] - self.band, s[0][0] + self.band, s[0][1] + self.band], + fill=255) + self._target_mask = np.asarray(im) > 0 + return float((ink & ~self._target_mask).sum() / ink.sum()) + + def judgements(self, pixels: np.ndarray) -> Dict[str, Optional[str]]: + frac = self.drawn_fraction() + ink = bool(ink_mask(pixels).any()) + stray = ink and self._stray(pixels) > self.stray_off_track + if self.complete: + progress = 4 + elif frac == 0 or not ink: + progress = 0 + else: + progress = 1 if frac < 0.4 else 2 if frac < 0.75 else 3 + if not ink: + drawn = "nothing" + elif self.complete and not stray and self.shapes and self.category in self.shapes: + drawn = self.category + else: + drawn = None + return {"progress": str(progress), "on_track": "off track" if stray else "on track", "drawn": drawn} + + def questions(self, task: str) -> Dict: + from laya.games import paint_judgements + + return paint_judgements(task, shapes=self.shapes) + + +def drawn_options(categories: Sequence[str] = TRAIN_CATEGORIES) -> Dict[str, str]: + """``drawn`` question options for doodle tasks: the categories, plus ``nothing``.""" + opts = {c: "a doodle of %s %s" % ("an" if c[0] in "aeiou" else "a", c) for c in categories} + opts["nothing"] = "nothing, the canvas is blank" + return opts + + +# -- rasterising for the independent classifier -------------------------------------------------------------------- +BITMAP = 28 + + +def to_bitmap(mask: np.ndarray, size: int = BITMAP) -> np.ndarray: + """A boolean ink mask cropped to its ink, padded to a square with a 10% border, and resized to ``size`` px: + ``(size, size)`` float32 in [0, 1]. Blank masks give zeros.""" + from PIL import Image + + ys, xs = np.nonzero(mask) + if not len(xs): + return np.zeros((size, size), np.float32) + y0, y1, x0, x1 = ys.min(), ys.max() + 1, xs.min(), xs.max() + 1 + h, w = y1 - y0, x1 - x0 + side = int(max(h, w) * 1.2) + 2 + sq = np.zeros((side, side), np.uint8) + oy, ox = (side - h) // 2, (side - w) // 2 + sq[oy:oy + h, ox:ox + w] = mask[y0:y1, x0:x1] * 255 + im = Image.fromarray(sq).resize((size, size), Image.BILINEAR) + return np.asarray(im, np.float32) / 255.0 + + +def render_strokes(strokes: Sequence[Sequence[Tuple[float, float]]], canvas: int = 256, + width: int = 4) -> np.ndarray: + """A 0-255 doodle drawn as the environment would draw it at ``DOODLE_FRAC`` of a 512 canvas: an ink mask.""" + from PIL import Image, ImageDraw + + im = Image.new("L", (canvas, canvas), 0) + d = ImageDraw.Draw(im) + for s in strokes: + if len(s) > 1: + d.line([tuple(p) for p in s], fill=255, width=width, joint="curve") + else: + x, y = s[0] + d.ellipse([x - width / 2, y - width / 2, x + width / 2, y + width / 2], fill=255) + return np.asarray(im) > 0 + + +def canvas_bitmap(pixels: np.ndarray) -> np.ndarray: + """The classifier's input for a canvas (RGB pixels).""" + return to_bitmap(ink_mask(pixels)) + + +def make_classifier(n_classes: int): + """A small CNN over ``BITMAP`` x ``BITMAP`` doodle bitmaps (the independent scorer).""" + import torch.nn as nn + + return nn.Sequential( + nn.Conv2d(1, 32, 3, padding=1), nn.ReLU(), nn.Conv2d(32, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.Conv2d(64, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Flatten(), nn.Dropout(0.3), nn.Linear(64 * 7 * 7, 256), nn.ReLU(), nn.Dropout(0.3), + nn.Linear(256, n_classes)) + + +def classifier_dataset(categories: Sequence[str], n_per: int, skip: int = 500, seed: int = 0): + """Bitmaps and labels for the classifier: ``n_per`` doodles per category after the first ``skip`` (which the + drawing episodes use), each rasterised with a random stroke width (3-6 px on a 256 canvas).""" + import random + + rng = random.Random(seed) + xs, ys = [], [] + for k, c in enumerate(categories): + for d in fetch_drawings(c, n_per, max_bytes=12_000_000, skip=skip): + xs.append(to_bitmap(render_strokes(d["strokes"], width=rng.randint(3, 6)))) + ys.append(k) + return np.stack(xs).astype(np.float32), np.array(ys, dtype=np.int64) + + +def train_classifier(categories: Sequence[str], n_per: int = 2000, epochs: int = 8, device: str = "cpu", + seed: int = 0, val_frac: float = 0.1) -> Tuple[Dict, Dict]: + """Train ``make_classifier`` on ``classifier_dataset``. Returns (checkpoint dict, metrics with held-back + top-1 accuracy overall and per category).""" + import torch + + torch.manual_seed(seed) + x, y = classifier_dataset(categories, n_per, seed=seed) + perm = np.random.default_rng(seed).permutation(len(y)) + n_val = int(len(y) * val_frac) + va, tr = perm[:n_val], perm[n_val:] + net = make_classifier(len(categories)).to(device) + opt = torch.optim.Adam(net.parameters(), lr=1e-3) + xt, yt = torch.from_numpy(x[tr])[:, None], torch.from_numpy(y[tr]) + for ep in range(epochs): + net.train() + order = torch.randperm(len(yt)) + for b in range(0, len(order), 256): + idx = order[b:b + 256] + loss = torch.nn.functional.cross_entropy(net(xt[idx].to(device)), yt[idx].to(device)) + opt.zero_grad() + loss.backward() + opt.step() + net.eval() + with torch.no_grad(): + pred = net(torch.from_numpy(x[va])[:, None].to(device)).argmax(-1).cpu().numpy() + per = {c: float((pred[y[va] == k] == k).mean()) for k, c in enumerate(categories) if (y[va] == k).any()} + metrics = {"n_train": int(len(tr)), "n_val": int(n_val), "val_accuracy": float((pred == y[va]).mean()), + "per_category": per, "epochs": epochs, "n_per": n_per} + ckpt = {"categories": list(categories), "state_dict": {k: v.cpu() for k, v in net.state_dict().items()}} + return ckpt, metrics + + +class DoodleScorer: + """Grades a canvas for a category with the independent classifier (``modal_app.py::train_quickdraw_classifier`` + writes the checkpoint: ``{"categories", "state_dict"}``). ``score`` is P(category); ``passed`` when it is the + classifier's top category. Usable as ``JSPaintEnv(scorer=...)``.""" + + def __init__(self, path: str, device: str = "cpu"): + import torch + + ck = torch.load(path, map_location=device) + self.categories = list(ck["categories"]) + self.net = make_classifier(len(self.categories)) + self.net.load_state_dict(ck["state_dict"]) + self.net.eval().to(device) + self.device = device + + def probabilities(self, pixels: np.ndarray) -> Dict[str, float]: + import torch + + x = torch.from_numpy(canvas_bitmap(pixels))[None, None].to(self.device) + with torch.no_grad(): + p = torch.softmax(self.net(x), -1)[0].cpu().numpy() + return {c: float(v) for c, v in zip(self.categories, p)} + + def __call__(self, pixels: np.ndarray, task: str) -> Dict: + ink = int(ink_mask(pixels).sum()) + if ink == 0: + return {"score": 0.0, "passed": False, "top": None, "ink": 0} + p = self.probabilities(pixels) + top = max(p, key=p.get) + return {"score": round(p.get(task, 0.0), 4), "passed": top == task, "top": top, + "top_p": round(p[top], 4), "ink": ink} + + +__all__ = ["BASE_URL", "TRAIN_CATEGORIES", "HELDOUT_CATEGORIES", "DOODLE_FRAC", "fetch_drawings", "doodle_strokes", + "stroke_length", "DoodleLabeller", "drawn_options", "to_bitmap", "render_strokes", "canvas_bitmap", + "BITMAP", "make_classifier", "classifier_dataset", "train_classifier", "DoodleScorer"] diff --git a/laya/vlm.py b/laya/vlm.py index 828688f4..2db76fcd 100644 --- a/laya/vlm.py +++ b/laya/vlm.py @@ -917,6 +917,7 @@ def __init__( self.cfg["readout"] = self.processor.laya_readout = self.model.readout self.cfg["option_attention"] = self.model.option_attention self.processor.laya_max_len = self.cfg.get("max_len", 1024) + self.processor.laya_head_max_len = self.cfg.get("head_max_len", 256) self.prep.check(self.processor) self.temperature = self.cfg.get("temperature", [1.0, 1.0, 1.0]) self.temperature_by_options = self.cfg.get("temperature_by_options", {}) diff --git a/laya/vlm_train.py b/laya/vlm_train.py index e4a78af5..b148cc31 100644 --- a/laya/vlm_train.py +++ b/laya/vlm_train.py @@ -120,10 +120,14 @@ def synthetic_examples(n: int = 8, seed: int = 0) -> List[Dict]: def make_item( processor, ex: Dict, rng: random.Random, shuffle: bool = True, max_len: Optional[int] = None, - head_max_len: int = 256, order: Optional[List[int]] = None, + head_max_len: Optional[int] = None, order: Optional[List[int]] = None, ) -> Dict: """Tokenize one example with a random (or the given) option order; the target is permuted to marker order. - ``max_len`` defaults to the agent's (``laya.vlm.build_vlm_inputs``).""" + ``max_len`` and ``head_max_len`` default to the agent's (``laya.vlm.build_vlm_inputs``; the agent leaves them on + the processor as ``laya_max_len`` / ``laya_head_max_len``), so training and evaluation build the same inputs as + ``predict``.""" + if head_max_len is None: + head_max_len = getattr(processor, "laya_head_max_len", 256) k = len(render_options(ex["q"])) if order is None: order = list(range(k)) diff --git a/mkdocs.yml b/mkdocs.yml index 5440cdc2..ab8aeba9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - Run jobs on Modal: how-to/run-on-modal.md - Evaluate a checkpoint: how-to/evaluate.md - Play games with a checkpoint: how-to/play-games.md + - Draw in JSPaint with the mouse: how-to/jspaint-env.md - Concepts: - Overview: concepts/overview.md diff --git a/modal_app.py b/modal_app.py index e1437699..2411df8e 100644 --- a/modal_app.py +++ b/modal_app.py @@ -407,6 +407,7 @@ def finetune_long( mix_alpha: float = 0.0, split_edge: int = 0, max_len: int = 0, + head_max_len: int = 0, max_train: int = 0, max_val: int = 0, restart: bool = False, @@ -450,8 +451,10 @@ def finetune_long( * ``split_edge`` > 0 turns on the processor's image splitting (``laya.preprocess``): each image is resized to that longest edge and cut into 512 tiles plus a global view, up to 17 views at 2048 against 1 without. Needs ``preprocess="processor"``. ``max_len`` (0: ``laya.vlm.default_max_len``, 1024 without splitting) - is the sequence cap; both are saved in the checkpoint. Neither applies with ``init_from``, which keeps the - checkpoint's. + is the sequence cap; both are saved in the checkpoint. ``split_edge`` does not apply with ``init_from``, + which keeps the checkpoint's. ``max_len`` and ``head_max_len`` (0: the checkpoint's, 256 for a fresh + head) do: they override the starting checkpoint's and are saved with the new one, so a task whose question + needs more than 256 tokens (the JSPaint drawing question, ~1,050) trains and plays uncut. * ``max_train`` / ``max_val`` > 0 keep the first records per dataset (file order) for a quicker run on a subset; ``max_train`` does not count the ``n_calib`` holdout. * Durability: every ``state_every_min`` minutes (clamped to ``vlm_train.MIN_STATE_MINUTES``, and backed off @@ -495,11 +498,13 @@ def finetune_long( % (max_passes or None, {n: round(epochs * len(train_ex) * probs[n] / sizes[n], 2) for n in names})) if init_from: - agent = VLMAgent(_ckpt_path(init_from), device="cuda") + budgets = {k: v for k, v in (("max_len", max_len), ("head_max_len", head_max_len)) if v} + agent = VLMAgent(_ckpt_path(init_from), device="cuda", **budgets) print("initialised from %s (temperatures %s)" % (init_from, [round(t, 3) for t in agent.temperature])) else: agent = VLMAgent(backbone=backbone, device="cuda", preprocess=preprocess, option_attention=option_attention, - image_split_edge=split_edge, **({"max_len": max_len} if max_len else {})) + image_split_edge=split_edge, **({"max_len": max_len} if max_len else {}), + **({"head_max_len": head_max_len} if head_max_len else {})) print("backbone %s, readout %s, preprocess %s, split_edge %d, max_len %d" % ( agent.cfg["backbone"], agent.model.readout, agent.prep.backend, agent.prep.split_edge, agent.cfg["max_len"])) init_temps = list(agent.temperature) @@ -515,6 +520,7 @@ def finetune_long( log = {"run": run_name, "args": dict(backbone=agent.cfg["backbone"], readout=agent.model.readout, datasets=datasets, val_datasets=val_datasets or datasets, preprocess=agent.prep.backend, split_edge=agent.prep.split_edge, max_len=agent.cfg["max_len"], + head_max_len=agent.cfg.get("head_max_len", 256), max_train=max_train, max_val=max_val, option_attention=agent.model.option_attention, w_ce_schedule=w_ce_schedule, mix=mix_weights, mix_alpha=mix_alpha, @@ -2540,3 +2546,470 @@ def full_eval(model: str, parts: str = ",".join(FULL_EVAL_PARTS), datasets: str print("saved", path, "on laya-checkpoints") if failed: raise SystemExit("full_eval: %s produced no results (see the errors above)" % ", ".join(failed)) + + +# --------------------------------------------------------------------------------------------------------------- +# JSPaint drawing task (laya/paintenv.py, laya/paintdata.py): labelled data from headless Chromium, and play eval. +# JSPaint is cloned at a pinned commit; Chromium comes from `playwright install --with-deps`. +JSPAINT_REPO = "https://github.com/r33drichards/jspaint" +JSPAINT_COMMIT = "53be67ab8c47cc0d2168899e7481bc04839c4c81" +paint_image = _with_local_code( + base_image.apt_install("git").pip_install("playwright==1.63.0").run_commands( + "playwright install --with-deps chromium", + "git clone %s /jspaint && cd /jspaint && git checkout %s" % (JSPAINT_REPO, JSPAINT_COMMIT), + ) +) + + +def _paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int, model: str, beta: float) -> list: + """Episodes for ``specs``: ``[seed, eps, random_prefix, messy]`` for the circle labeller, or with two more items + ``[..., category, doodle_index]`` for a Quick, Draw! doodle (``laya.quickdraw.DoodleLabeller``).""" + from laya.paintdata import play_labelled_episode + from laya.quickdraw import DoodleLabeller, drawn_options, fetch_drawings + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, ModelPolicy + + os.makedirs(os.path.join(tmp_dir, "images"), exist_ok=True) + + def save_frame(name, img): + rel = "images/%s.png" % name + img.save(os.path.join(tmp_dir, rel), optimize=True) + return rel + + driver = None + if model: + from laya.vlm import VLMAgent + + agent = VLMAgent(_ckpt_path(model), device="cuda", head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN) + driver = ModelPolicy(agent, judge=False) + doodles = {} + for sp in specs: + if len(sp) > 4: + doodles[sp[4]] = max(doodles.get(sp[4], 0), sp[5] + 1) + doodles = {c: fetch_drawings(c, n) for c, n in doodles.items()} + shapes = drawn_options() + out, summaries = os.path.join(tmp_dir, "%s-%03d.jsonl" % (split, shard)), [] + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, executable_path=None) as env, \ + open(out + ".part", "w") as f: + for sp in specs: + seed, eps, prefix, messy = sp[:4] + task, labeller, env.max_steps = "circle", None, 260 + if len(sp) > 4: + task, doodle = sp[4], doodles[sp[4]][sp[5]] + labeller, env.max_steps = DoodleLabeller(task, doodle, shapes), 400 + env.task = task + if driver is not None: + driver.task = task + recs, summ = play_labelled_episode(env, seed, "%s-%d" % (split, seed), save_frame, eps=eps, + random_prefix=prefix, judge_every=judge_every, messy=messy, + driver=driver, beta=beta, task=task, labeller=labeller) + if len(sp) > 4: + summ.update(task=task, doodle=doodle["key_id"]) + f.writelines(json.dumps(r) + "\n" for r in recs) + summaries.append(summ) + os.rename(out + ".part", out) + data_vol.commit() + return summaries + + +@app.function(image=paint_image, cpu=2, memory=4096, timeout=2 * 60 * 60, volumes={"/data": data_vol}, + retries=modal.Retries(max_retries=2, initial_delay=5.0)) +def paint_shard(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int = 3) -> list: + """Play the episodes in ``specs`` (``[seed, eps, random_prefix, messy]``) with the labeller driving and write + ``-.jsonl`` plus the frames under ``tmp_dir``. Returns one summary per episode.""" + return _paint_shard(tmp_dir, split, shard, specs, judge_every, "", 1.0) + + +@app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=3 * 60 * 60, + volumes={"/data": data_vol, "/cache/hf": hf_vol, "/ckpt": ckpt_vol.read_only()}, + retries=modal.Retries(max_retries=2, initial_delay=5.0)) +def paint_shard_dagger(tmp_dir: str, split: str, shard: int, specs: list, judge_every: int = 3, model: str = "", + beta: float = 0.5) -> list: + """``paint_shard`` with a trained checkpoint driving each step with probability ``1 - beta`` (DAgger).""" + return _paint_shard(tmp_dir, split, shard, specs, judge_every, model, beta) + + +def _get_retrying(call, tries: int = 8): + """``call.get()``, retrying transient client/connection errors (a lost connection while waiting on a long + fan-out cancelled a 48-shard prep once); errors raised by the function itself are not retried.""" + for k in range(tries): + try: + return call.get() + except (modal.exception.ConnectionError, TimeoutError, OSError) as e: + if k == tries - 1: + raise + print("waiting on %s again after %r" % (call.object_id, e), flush=True) + time.sleep(10 * (k + 1)) + + +@app.function(image=paint_image, cpu=2, memory=8192, timeout=3 * 60 * 60, volumes={"/data": data_vol}) +def prepare_paint(name: str = "paint_circle_v2", n_train: int = 240, n_val: int = 24, shards: int = 32, seed: int = 0, + judge_every: int = 3, model: str = "", beta: float = 0.5, categories: str = "", + per_category: int = 10, val_per_category: int = 2, resume: bool = False): + """Write /data/vqa//{train,val}.jsonl + images/: labelled JSPaint circle episodes. Refuses to replace an + existing prepared dataset (they are create-only; ``paint_circle`` was the first, fixed-target labeller). + + Each episode runs a behaviour policy in headless Chromium: the state-based labeller (``laya.paintdata``) with + probability ``1 - eps``, else a random action, after ``random_prefix`` purely random steps. ``eps`` is drawn from + (0, 0.1, 0.2, 0.3) and ``random_prefix`` from (0, 0, 10, 30, 60), so the data covers clean circles, noisy ones, + and recoveries from another start; a third of the episodes with a random start are ``messy`` (the random steps + may draw, leaving stray ink). Every step is labelled with the labeller's action for that state (soft + over the neighbouring compass points); every ``judge_every`` steps also with progress, on track and what is + drawn. Train and val use disjoint seeds. Episodes are split over ``shards`` containers. + + With ``model`` (a run under /ckpt), the checkpoint drives each step with probability ``1 - beta`` (DAgger, on L4 + containers): the data then covers the states the model itself reaches, labelled with what the labeller would + do there. Use a new ``seed`` per round so the episodes differ from the earlier sets'. + + With ``categories`` (comma list of Quick, Draw! categories, or ``train`` for ``laya.quickdraw.TRAIN_CATEGORIES``) + the episodes are doodles instead: ``per_category`` train and ``val_per_category`` val episodes per category, each + following a different human doodle (train uses the category's doodles 0..per_category-1, val the next ones; + the independent classifier trains on doodles from 500 on). ``n_train`` / ``n_val`` are then ignored. + + Shards run as independent calls; waiting on them retries a lost connection (``_get_retrying``). With + ``resume`` an earlier attempt's ``.tmp`` directory is kept and shards whose file is already there are skipped + (the same arguments give the same shards). + """ + import random + import shutil + from collections import Counter + + final_dir = "/data/vqa/" + name + if os.path.exists(final_dir): + raise SystemExit("%s exists; prepared datasets are create-only, pass a new --name" % final_dir) + tmp_dir = final_dir + ".tmp" + if not resume: + shutil.rmtree(tmp_dir, ignore_errors=True) + os.makedirs(os.path.join(tmp_dir, "images"), exist_ok=True) + data_vol.commit() + rng = random.Random(seed) + from laya.quickdraw import TRAIN_CATEGORIES + + cats = [c.strip() for c in categories.split(",") if c.strip()] + cats = list(TRAIN_CATEGORIES) if cats == ["train"] else cats + jobs = [] + for split, n, seed0 in (("train", n_train, seed), ("val", n_val, seed + 1_000_000)): + specs = [] + if cats: + per, first = (per_category, 0) if split == "train" else (val_per_category, per_category) + for ci, c in enumerate(cats): + for d in range(per): + prefix = rng.choice((0, 0, 0, 10, 30)) + specs.append([seed0 + ci * 1000 + d, rng.choice((0.0, 0.05, 0.1, 0.2)), prefix, + bool(prefix) and rng.random() < 1 / 3, c, first + d]) + rng.shuffle(specs) + for i in range(0 if cats else n): + prefix = rng.choice((0, 0, 10, 30, 60)) + specs.append([seed0 + i, rng.choice((0.0, 0.1, 0.2, 0.3)), prefix, bool(prefix) and rng.random() < 1 / 3]) + k = max(1, min(shards, n)) + for s in range(k): + if specs[s::k]: + jobs.append((tmp_dir, split, s, specs[s::k], judge_every) + ((model, beta) if model else ())) + summaries = {"train": [], "val": []} + shard_fn = paint_shard_dagger if model else paint_shard + done_files = set(os.listdir(tmp_dir)) + todo = [j for j in jobs if "%s-%03d.jsonl" % (j[1], j[2]) not in done_files] + print("%d shards, %d already written" % (len(jobs), len(jobs) - len(todo)), flush=True) + calls = [(job, shard_fn.spawn(*job)) for job in todo] + for job, call in calls: + res = _get_retrying(call) + split = job[1] + summaries[split].extend(res) + data_vol.reload() + meta = {"source": "JSPaint %s@%s in headless Chromium, %s" % ( + JSPAINT_REPO, JSPAINT_COMMIT, "Quick, Draw! doodles followed by laya/quickdraw.py DoodleLabeller " + "(quickdraw_dataset simplified ndjson, CC BY 4.0)" if cats else + "state-based circle labeller (laya/paintdata.py)"), + "judge_every": judge_every, "driver": model or "labeller", "beta": beta if model else None, + "categories": cats or None} + for split in ("train", "val"): + parts = sorted(fn for fn in os.listdir(tmp_dir) if fn.startswith(split + "-") and fn.endswith(".jsonl")) + kinds = Counter() + with open(os.path.join(tmp_dir, split + ".jsonl"), "w") as out: + for fn in parts: + with open(os.path.join(tmp_dir, fn)) as f: + for line in f: + kinds[json.loads(line)["id"].rsplit("-", 1)[1]] += 1 + out.write(line) + os.remove(os.path.join(tmp_dir, fn)) + eps = summaries[split] + meta[split] = {"episodes": len(eps), "records": dict(kinds), + "labeller_mean_verifier": round(sum(e["verifier"] for e in eps) / max(1, len(eps)), 4), + "mean_steps": round(sum(e["steps"] for e in eps) / max(1, len(eps)), 1)} + print(split, meta[split]) + with open(os.path.join(tmp_dir, "meta.json"), "w") as f: + json.dump(meta, f, indent=2) + with open(os.path.join(tmp_dir, "episodes.json"), "w") as f: + json.dump(summaries, f) + sources = {JSPAINT_REPO: JSPAINT_COMMIT} + if cats: + from laya.quickdraw import BASE_URL + + sources[BASE_URL % ""] = "first recognised doodles per category, CC BY 4.0" + _write_manifest(tmp_dir, sources, + dict(n_train=n_train, n_val=n_val, shards=shards, seed=seed, judge_every=judge_every, + model=model, beta=beta, categories=cats, per_category=per_category, + val_per_category=val_per_category)) + os.rename(tmp_dir, final_dir) + open(os.path.join(final_dir, "_READY"), "w").close() + data_vol.commit() + return meta + + +@app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=3 * 60 * 60, + volumes={"/cache/hf": hf_vol, "/ckpt": ckpt_vol}) +def paint_eval(model: str = "cauldron-score-2ep-bidir-full/best", episodes: int = 5, seed: int = 900_000, + policies: str = "model,labeller,expert,random", judge_stop: bool = False, name: str = "", + task: str = "circle"): + """Play the JSPaint circle task with a checkpoint (a run under /ckpt, or a Hub id) and the reference policies: + ``labeller`` (the state-based labeller the training data comes from), ``expert`` (the older scripted circle + centred on the canvas) and ``random``. + + The model runs with the raised token budgets (``paintenv.PAINT_HEAD_MAX_LEN`` / ``PAINT_MAX_LEN``) or the + checkpoint's if larger, the full drawing question, two-frame state and the judgement questions each step. + Seeds start at 900,000, disjoint from ``prepare_paint``'s. Each episode's final frame and a GIF go to + ``/paint_eval//-seed.{png,gif}``. The summary (verifier score, pass rate, steps, + action mix and per-step judgements) is written to ``/paint_eval/.json`` for a run on the + volume and returned. + """ + from laya.paintdata import LabellerPolicy + from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, + model_policy, play_episodes, random_policy) + from laya.vlm import VLMAgent + + path = _ckpt_path(model) + src = path if os.path.exists(os.path.join(path, "vlm_agent_config.json")) else model + probe = VLMAgent(src, device="cuda") + budgets = dict(head_max_len=max(PAINT_HEAD_MAX_LEN, probe.cfg.get("head_max_len", 256)), + max_len=max(PAINT_MAX_LEN, probe.cfg.get("max_len", 1024))) + del probe + agent = VLMAgent(src, device="cuda", **budgets) + names = [p for p in policies.split(",") if p] + out = {"model": model, "task": task, "budgets": budgets, "episodes": episodes, "seed": seed, + "judge_stop": judge_stop, + "jspaint": JSPAINT_COMMIT, "policies": {}} + on_volume = os.path.exists(os.path.join(path, "vlm_agent_config.json")) + img_dir = os.path.join(path, "paint_eval", name or time.strftime("%Y%m%d-%H%M%S")) + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, task=task, canvas_size=agent.prep.image_size, + keep_frames=True) as env: + for pname in names: + pol = {"model": lambda: model_policy(agent, task, judge_stop=judge_stop), "expert": circle_expert, + "labeller": LabellerPolicy, "random": lambda: random_policy(seed)}[pname]() + steps_log = [] + + def on_step(env, action, pol=pol, pname=pname, steps_log=steps_log): + if env.done and on_volume: # the final frame and the whole episode, for looking at + os.makedirs(img_dir, exist_ok=True) + stem = os.path.join(img_dir, "%s-seed%d" % (pname, env.seed)) + env.frames[-1].save(stem + ".png") + env.frames[0].save(stem + ".gif", save_all=True, append_images=env.frames[1:], duration=80, + loop=0) + if pname == "model": + steps_log.append({"seed": env.seed, "step": env.steps, "action": action, + "top_p": max(pol.last["probabilities"].values()), + "act_probability": pol.last["action"]["act_probability"], + **({"judgement": {k: v for k, v in pol.last_judgement.items() + if not k.endswith("_probs")}} if pol.last_judgement else {})}) + + t = time.time() + res = play_episodes(env, pol, episodes, seed=seed, on_step=on_step) + res["seconds"] = round(time.time() - t, 1) + if pname == "model": + res["provenance"] = pol.provenance + res["steps_log"] = steps_log + out["policies"][pname] = res + print("%-7s mean score %.3f pass rate %.2f mean steps %.1f actions %s (%.0fs)" % ( + pname, res["mean_score"], res["pass_rate"], res["mean_steps"], + dict(sorted(res["actions"].items(), key=lambda kv: -kv[1])[:6]), res["seconds"]), flush=True) + if on_volume: + d = os.path.join(path, "paint_eval") + os.makedirs(d, exist_ok=True) + fn = os.path.join(d, (name or time.strftime("%Y%m%d-%H%M%S")) + ".json") + with open(fn, "w") as f: + json.dump(out, f, indent=2, default=str) + ckpt_vol.commit() + print("wrote", fn) + return {k: v for k, v in out.items() if k != "policies"} | { + "summary": {p: {k: r[k] for k in ("mean_score", "pass_rate", "mean_steps", "actions")} + for p, r in out["policies"].items()}} + + +QUICKDRAW_CLASSIFIER = "/ckpt/quickdraw/classifier.pt" + + +@app.function(image=image, gpu="L4", cpu=8, memory=32768, timeout=2 * 60 * 60, volumes={"/ckpt": ckpt_vol}) +def train_quickdraw_classifier(n_per: int = 3000, epochs: int = 12, seed: int = 0): + """Train the independent doodle scorer (``laya.quickdraw.train_classifier``) on the training and held-out + categories and write it to ``/ckpt/quickdraw/classifier.pt`` (with ``classifier-metrics.json``). Its doodles + are each category's recognised doodles from 500 on, disjoint from the ones the drawing episodes follow. It is + what ``doodle_eval`` grades drawings with, so the drawing model is never graded by its own judgement.""" + import torch + + from laya.quickdraw import HELDOUT_CATEGORIES, TRAIN_CATEGORIES, train_classifier + + cats = list(TRAIN_CATEGORIES + HELDOUT_CATEGORIES) + ckpt, metrics = train_classifier(cats, n_per=n_per, epochs=epochs, device="cuda", seed=seed) + os.makedirs(os.path.dirname(QUICKDRAW_CLASSIFIER), exist_ok=True) + if os.path.exists(QUICKDRAW_CLASSIFIER): + raise SystemExit("%s exists; checkpoints are create-only" % QUICKDRAW_CLASSIFIER) + torch.save(ckpt, QUICKDRAW_CLASSIFIER) + with open(os.path.join(os.path.dirname(QUICKDRAW_CLASSIFIER), "classifier-metrics.json"), "w") as f: + json.dump(metrics, f, indent=2) + ckpt_vol.commit() + print("val accuracy %.3f over %d categories" % (metrics["val_accuracy"], len(cats))) + return metrics + + +@app.function(image=paint_image, gpu="L4", cpu=4, memory=16384, timeout=2 * 60 * 60, + volumes={"/cache/hf": hf_vol, "/ckpt": ckpt_vol}) +def doodle_eval_task(model: str, task: str, episodes: int, seed: int, policies: str, first_doodle: int, + decide: str = "argmax") -> dict: + """``doodle_eval`` for one category: play ``episodes`` with each policy, graded by the classifier.""" + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, model_policy, \ + play_episodes, random_policy + from laya.quickdraw import DoodleLabeller, DoodleScorer, drawn_options, fetch_drawings + + scorer = DoodleScorer(QUICKDRAW_CLASSIFIER, device="cuda") + out = {"task": task, "policies": {}} + agent = None + if "model" in policies: + from laya.vlm import VLMAgent + + agent = VLMAgent(_ckpt_path(model), device="cuda", head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN) + doodles = fetch_drawings(task, first_doodle + episodes)[first_doodle:] + with JSPaintServer("/jspaint") as server, JSPaintEnv(server.url, task=task, max_steps=400, scorer=scorer, + keep_frames=True) as env: + for pname in policies.split(","): + if pname == "model": + pol = model_policy(agent, task, judge=False, decide=decide, seed=seed) + elif pname == "random": + pol = random_policy(seed) + else: # the labeller following held-back human doodles: what a faithful copier would score + labellers = [DoodleLabeller(task, d, drawn_options()) for d in doodles] + + def pol(env, labellers=labellers): + lab = labellers[env.episode % len(labellers)] + if getattr(lab, "env", None) is not env or lab._episode != env.episode: + lab.reset(env) + lab._episode = env.episode + return lab.action(env, env.visible_pixels()) + res = play_episodes(env, pol, episodes, seed=seed) + out["policies"][pname] = {k: res[k] for k in ("mean_score", "pass_rate", "mean_steps")} | { + "episodes": [{k: e.get(k) for k in ("seed", "score", "passed", "top", "steps", "ended")} + for e in res["results"]]} + if pname == "model" and _ckpt_exists(model): + d = os.path.join(_ckpt_path(model), "doodle_eval_frames") + os.makedirs(d, exist_ok=True) + env.frames[-1].save(os.path.join(d, "%s-last.png" % task.replace(" ", "_"))) + ckpt_vol.commit() + return out + + +def _ckpt_exists(model: str) -> bool: + return os.path.exists(os.path.join(_ckpt_path(model), "vlm_agent_config.json")) + + +@app.function(image=image, timeout=4 * 60 * 60, volumes={"/ckpt": ckpt_vol}) +def doodle_eval(model: str = "paint-circle-v3/best", tasks: str = "train,heldout", episodes: int = 2, + seed: int = 910_000, policies: str = "model,labeller,random", first_doodle: int = 200, + name: str = "", decide: str = "argmax"): + """Ask a checkpoint to draw each Quick, Draw! category in JSPaint and grade it with the independent classifier + (``train_quickdraw_classifier``): score = P(requested category), passed = the classifier's top category. + + ``tasks`` takes category names and ``train`` / ``heldout`` (``laya.quickdraw``). Each category runs in its own + L4 container. ``labeller`` follows held-back human doodles (from ``first_doodle`` on, unseen by training and by + the classifier), which is roughly the ceiling; ``random`` is the floor. Results are summarised for the training + and the held-out categories and written to ``/doodle_eval/.json``.""" + from laya.quickdraw import HELDOUT_CATEGORIES, TRAIN_CATEGORIES + + names = [] + for t in tasks.split(","): + t = t.strip() + names += list(TRAIN_CATEGORIES) if t == "train" else list(HELDOUT_CATEGORIES) if t == "heldout" else [t] + jobs = [(model, t, episodes, seed, policies, first_doodle, decide) for t in names] + per_task = {r["task"]: r for r in doodle_eval_task.starmap(jobs)} + summary = {} + for group, cats in (("train", TRAIN_CATEGORIES), ("heldout", HELDOUT_CATEGORIES)): + rows = [per_task[c] for c in cats if c in per_task] + if not rows: + continue + summary[group] = {p: {"mean_score": round(sum(r["policies"][p]["mean_score"] for r in rows) / len(rows), 4), + "pass_rate": round(sum(r["policies"][p]["pass_rate"] for r in rows) / len(rows), 4)} + for p in policies.split(",")} + out = {"model": model, "decide": decide, "episodes_per_task": episodes, "seed": seed, "first_doodle": first_doodle, + "summary": summary, "tasks": per_task} + print(json.dumps(summary, indent=2)) + for t in names: + print("%-12s %s" % (t, " ".join("%s %.2f/%.0f%%" % (p, per_task[t]["policies"][p]["mean_score"], + 100 * per_task[t]["policies"][p]["pass_rate"]) + for p in policies.split(",")))) + if _ckpt_exists(model): + ckpt_vol.reload() + d = os.path.join(_ckpt_path(model), "doodle_eval") + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, (name or time.strftime("%Y%m%d-%H%M%S")) + ".json"), "w") as f: + json.dump(out, f, indent=2) + ckpt_vol.commit() + return summary + + +@app.function(image=image, gpu="L4", cpu=8, memory=32768, timeout=2 * 60 * 60, + volumes={"/cache/hf": hf_vol, "/data": data_vol.read_only(), "/ckpt": ckpt_vol}) +def prompt_sensitivity(model: str, dataset: str, tasks: str = "circle,square,triangle,line", n: int = 200, + name: str = "", seed: int = 0): + """Does the drawing policy read the task? For ``n`` action records from ``dataset``'s val split (states whose + task is one of ``tasks``), ask the action question once per task prompt, same state, and report: + + * ``tv``: mean total-variation distance between the action distributions under two different prompts (0: the + prompt changes nothing; 1: disjoint); + * ``p_label_true`` / ``p_label_other``: mean probability of the labelled action under the state's own prompt + and under the other prompts. A model that uses the task gives the label more probability with the right one. + Written to ``/prompt_sensitivity/.json`` for a run on the volume.""" + import itertools + import random + + from PIL import Image + + from laya.games import paint_goal, paint_question + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN + from laya.vlm import VLMAgent + + names = [t.strip() for t in tasks.split(",") if t.strip()] + root = "/data/vqa/" + dataset + recs = [] + with open(os.path.join(root, "val.jsonl")) as f: + for line in f: + r = json.loads(line) + if not r["id"].endswith("-action"): + continue + own = [t for t in names if paint_goal(t) in r["question"]["instructions"]] + if own: + recs.append((r, own[0])) + random.Random(seed).shuffle(recs) + recs = recs[:n] + agent = VLMAgent(_ckpt_path(model), device="cuda", head_max_len=PAINT_HEAD_MAX_LEN, max_len=PAINT_MAX_LEN) + qs = {t: paint_question(t)["action"] for t in names} + tv, p_true, p_other, rows = [], [], [], [] + for r, own in recs: + state = {"images": [Image.open(os.path.join(root, p)).convert("RGB") for p in r["images"]], + "context": r["state_text"]} + out = agent.predict(state, qs, strict=True)["answers"] + dist = {t: out[t]["probabilities"] for t in names} + label = list(qs[own]["criteria"])[r["label"]] + for a, b in itertools.combinations(names, 2): + tv.append(0.5 * sum(abs(dist[a][k] - dist[b][k]) for k in dist[a])) + p_true.append(dist[own][label]) + p_other.extend(dist[t][label] for t in names if t != own) + rows.append({"id": r["id"], "task": own, "label": label, + "top": {t: max(dist[t], key=dist[t].get) for t in names}}) + out = {"model": model, "dataset": dataset, "tasks": names, "n": len(recs), + "tv": round(sum(tv) / len(tv), 4), "p_label_true": round(sum(p_true) / len(p_true), 4), + "p_label_other": round(sum(p_other) / len(p_other), 4), + "top_differs": round(sum(len(set(x["top"].values())) > 1 for x in rows) / len(rows), 4)} + print(json.dumps(out, indent=2)) + if _ckpt_exists(model): + d = os.path.join(_ckpt_path(model), "prompt_sensitivity") + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, (name or dataset) + ".json"), "w") as f: + json.dump(dict(out, rows=rows), f, indent=2) + ckpt_vol.commit() + return out diff --git a/pyproject.toml b/pyproject.toml index 9a257ca2..6557e710 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,9 @@ dependencies = [ "numpy>=1.20.0", ] +[project.optional-dependencies] +paint = ["playwright", "pillow"] + [project.urls] Homepage = "https://github.com/r33drichards/laya-vision" Model = "https://huggingface.co/thaitea/laya-vision-smolvlm-256m" diff --git a/site-docs/how-to/jspaint-env.md b/site-docs/how-to/jspaint-env.md new file mode 100644 index 00000000..07d12483 --- /dev/null +++ b/site-docs/how-to/jspaint-env.md @@ -0,0 +1,240 @@ +# Draw in JSPaint with the mouse + +This environment gives the model a real paint program, [JSPaint](https://github.com/r33drichards/jspaint), and a +task such as "draw a circle". The model can only use the mouse. Each step it sees a screenshot of the canvas and +picks one mouse action, which is sent to the app as real pointer events. When the episode ends, a verifier scores +the pixels on the canvas. + +The environment is in `laya/paintenv.py`, the verifier in `laya/circle_verifier.py`, and the runner in +[`examples/jspaint_circle.py`](https://github.com/r33drichards/laya-vision/blob/main/examples/jspaint_circle.py). + +## Run it + +```bash +pip install -e ".[paint]" +git clone https://github.com/r33drichards/jspaint ../jspaint +python examples/jspaint_circle.py --jspaint ../jspaint --policies expert,random # reference points, no model +python examples/jspaint_circle.py --jspaint ../jspaint --episodes 5 --revision # + thaitea/laya-vision +``` + +JSPaint needs no build step. `JSPaintServer` serves the checkout as static files, and Playwright drives it in +headless Chromium. Set `LAYA_CHROMIUM` (or pass `--chromium`) to use an existing Chromium binary instead of +running `playwright install`. + +Each run writes a new directory under `results/jspaint/`. It contains: + +- `summary.json`: per-policy mean score, pass rate, and the verifier components for every episode; +- one folder per policy and episode, each holding: + - `trajectory.jsonl`: action, cursor, pen state, and the model's probabilities over the actions; + - `final.png`: the last frame; + - `episode.gif`: every frame of the episode. + +## Observation and actions + +Each episode starts the same way. The canvas is cleared to white, the Brush tool is selected (4 px strokes), and +the cursor is placed at a seeded point in the middle of the canvas. + +| | | +|---|---| +| Canvas | 512×512, the size of laya-vision's input image (`--canvas-size`, default: the model's `image_size`). | +| Observation | Two screenshots of the canvas, 8 steps ago (`frame_gap`) and now, with the cursor drawn on both: a red ring and cross when the button is up, a filled red dot when it is held down. The state text gives the pen state, the step count and the last 12 actions (`history`). `JSPaintEnv.state()` builds it. | +| Actions (`choice`) | 32 compass moves, `N` `NbE` `NNE` `NEbN` `NE` … `NbW`, one every 11.25°. Each moves 6 px (`step_px`) and draws while the pen is down. Each option's text gives its bearing in degrees. The other three are `PEN_DOWN`, `PEN_UP` and `DONE`. | +| End | The episode ends on `DONE` or after 260 steps (`max_steps`). The pen is released automatically at the end. | +| Reward | The verifier score, paid at the end. With `reward="shaped"`, each step instead earns the change in score. | + +### Why the canvas is 512×512 + +The released checkpoints read every image as a single 512×512 view, and the processor resizes to that square +without keeping the aspect ratio. JSPaint's default 683×384 canvas was therefore squashed: a drawn circle reached the +model as a tall ellipse (165×293 px). On that canvas laya-vision called a perfect ring "blank", with P(circle) 0.05. + +A 512×512 canvas reaches the vision tower pixel for pixel. On it, the model recognizes the scripted expert's circles +without any cropping, and still tells them apart from a square drawn the same way: + +| Canvas (JSPaint, 512×512) | "What is drawn?" (6 options) | "Which shape?" (5 options) | Verifier | +|---|---|---|---| +| Expert circle, seed 0 | circle (0.67) | circle (0.94) | 0.97 | +| Expert circle, seed 1 | circle (0.84) | circle (0.94) | 0.97 | +| Expert circle, seed 2 | circle (0.82) | circle (0.94) | 0.97 | +| Square drawn with the same moves | square (P(circle) 0.09) | square (0.03) | 0.00 | +| Blank | blank (0.08) | heart (0.15) | 0.00 | + +Each cell is the top answer, with the probability of "circle" in brackets. These are spot checks of +`thaitea/laya-vision` at revision `f2fe3c1`, not a benchmark. Two caveats. The yes/no question "Does the image +show a circle?" stays below 0.35 on all of them, so ask it as a choice. And the red cursor marker in the +observation sometimes pulls the 6-option answer to "line". + +JSPaint reads its canvas size from `localStorage` at start-up, so the environment writes `width` and `height` +there before the page loads. JSPaint itself is unmodified. The browser viewport is 900×720 so the whole canvas is +on screen. + +### Move size + +The model can only pick from a list of options, so the actions are relative moves. They are fine-grained because +coarse moves can't draw a round circle. Scripted-expert scores over 3 episodes, measured on the earlier 683×384 +canvas (radius 115 px): + +| Directions | Step | Expert score | Steps per circle | +|---|---|---|---| +| 8 | 24 px | 0.39 (an octagon; fails) | 42 | +| 16 | 12 px | 0.83 | 78 | +| 16 | 8 px | 0.91 | 116 | +| 32 | 8 px | 0.94 | 116 | +| **32** | **6 px (default)** | **0.95** | **139** | +| 32 | 4 px | 0.96 | 227 | + +More directions matter most, because the heading error per step falls from 22.5° to 5.6°. Steps below 6 px add +little and make episodes longer. Both settings are configurable (`directions=8|16|32`, `step_px`; `--directions`, +`--step-px` on the command line). + +### Memory and the full question + +One screenshot does not show which way the stroke was heading or what the model just tried, so each step carries +an earlier frame and the recent actions as well. + +The drawing question is sent in full. It has 35 options, each with its own description, and comes to about 1,050 +tokens. The checkpoint's defaults (`head_max_len` 256, `max_len` 1024) would cut every option and half the +instructions, so the model is loaded with `head_max_len=PAINT_HEAD_MAX_LEN` (1,536) and `max_len=PAINT_MAX_LEN` +(2,560). `ModelPolicy` calls `predict(..., strict=True)`, which raises instead of cutting. The model runs before +this change were affected: every option and 48 of the 82 instruction tokens had been cut. + +The screenshot has no OS cursor, so the cursor is drawn onto it. This is the same approach as Maze and Snake. + +The discrete actions are built on a mouse-only tool API: `move_mouse(dx, dy)`, `mouse_down()`, `mouse_up()` and +`screenshot()`. `paintenv.TOOLS` describes these tools as JSON-schema tool definitions, and +`JSPaintEnv.call_tool(name, args)` runs them. A tool-calling agent can therefore drive the same environment with +free coordinates. + +## Judgement questions + +Each step the model also answers three judgement questions (`laya.games.paint_judgements`) about the same state, +in the same `predict` call as the action: + +| Question | Type | Asks | +|---|---|---| +| `progress` | score, 0–4 | How far the canvas is toward the task, from "nothing useful drawn yet" to "the task is complete" | +| `on_track` | choice | "on track" or "off track": could what is drawn so far become the task by continuing? | +| `drawn` | choice | What the canvas shows: circle, oval, arc, line, square, triangle, scribble or nothing | + +`on_track` uses two named options rather than a yes/no (`noul`) question, because on line drawings the model's +yes/no answers were unreliable and named choices were not. + +The runner logs these and the checkpoint's act-vs-escalate gate (`act_probability`) for every step. At the end of +each episode it asks `drawn` about the clean final canvas, with no cursor, and saves the answer as the episode's +`judged_final` label. That label is what lets a failed attempt be relabelled as an example of what was actually +drawn. With `--judge-stop`, the model's own progress judgement ends the episode: it answers `DONE` once the pen is +up and it rates the task complete. + +### Do the judgements track reality? + +[`examples/jspaint_judge_check.py`](https://github.com/r33drichards/laya-vision/blob/main/examples/jspaint_judge_check.py) +asks the judgements every 10 steps along the scripted expert's circles and along random play. It compares them with +true progress: the share of the 36 sectors around the target circle's centre that the pen has passed through. For +zero-shot `thaitea/laya-vision` (revision `f2fe3c1`, 3 episodes each): + +| True progress (expert) | Checks | Judged progress (0–4) | P(on track) | +|---|---|---|---| +| 0 (not drawing yet) | 4 | 1.61 | 0.27 | +| 0–25% | 11 | 1.50 | 0.26 | +| 25–50% | 12 | 1.55 | 0.27 | +| 50–75% | 12 | 1.57 | 0.26 | +| 75–100% | 16 | 1.68 | 0.28 | +| Random policy, all checks | 69 | 1.70 | 0.30 | + +`drawn` works: it labelled the clean final canvas of all 3 expert circles "circle". `progress` and `on_track` do +not work zero-shot. Both are nearly flat, with a rank correlation of 0.40 between true and judged progress over a +narrow range, and random play is rated slightly higher than the expert. The environment knows the true progress, +so these questions can be trained with labels it produces, alongside the actions. + +## The circle verifier + +`score_circle(pixels)` reads the canvas pixels from the page with `toDataURL`, so the cursor overlay is never +counted as ink. It fits a least-squares circle to the ink and computes five components, each between 0 and 1: + +- **roundness**: how constant the radius is around the ring. It is `1 - std(r_k) / (0.05 r)`, where `r_k` is the + mean radius of the ink in each of 36 angular sectors. Averaging within a sector cancels the stroke width, so this + measures the shape itself: a clean ring scores 0.99, a 16-sided polygon 0.88, an octagon 0.56, and a square or a + 4:3 ellipse 0; +- **coverage**: the share of 36 angular sectors that contain ink on the ring; +- **clean**: the share of ink within `0.2 r` of the ring, so scribbles and fills count against the drawing; +- **closure**: 1 if no gap in the ring is wider than 30°, falling to 0 at a 120° gap; +- **size**: 1 if the radius is between 8% and 50% of the canvas height and the centre is on the canvas, else 0. + +`score` is the product of the five. An episode `passed` if the score is at least 0.7 and the circle is closed. + +`tests/test_circle_verifier.py` checks both directions: + +- These pass: a drawn ring, a small ring, a coloured ring, and 16- and 32-sided polygons (what the fine moves draw). +- These all fail: an octagon, a line, a dot, a tiny ring, a half arc, a three-quarter arc, a square, a flat ellipse, + a filled disc, a ring with a stray line, and random scribbles. + +## Reference points + +`circle_expert()` is a scripted policy. It walks with the pen up straight to the nearest point of a circle centred +on the canvas, and presses the button once it is within half a step of the radius. It then traces one full turn, +each step taking the forward move that lands closest to the radius. `random_policy` picks actions uniformly. On +seeds 900000–900002, on the 512×512 canvas with the default 32 directions and 6 px steps: + +| Policy | Mean score | Pass rate | Mean steps | +|---|---|---|---| +| Scripted expert | 0.97 | 3/3 | 181.7 | +| Random | 0.00 | 0/3 | 22.0 | +| `thaitea/laya-vision`, zero-shot | 0.00 | 0/3 | 31.0 | + +With the full question and two-frame memory, the zero-shot checkpoint moves the cursor (mostly `NNW`, 81 of 93 +actions) but never presses the button, and ends each episode with `DONE` within 6–48 steps. Its top option gets +about 0.05 probability against 0.03 for uniform, so it is close to guessing. Recognising a circle is not the same as knowing how to draw one: it was never trained on this task. The expert's trajectories (screenshot, action) are the obvious data to +train it on, as the game checkpoints were trained on expert frames. + +## Training on Modal + +Three jobs in `modal_app.py` train and test the drawing policy: + +1. **`prepare_paint`** generates the labelled episodes in headless Chromium, over parallel containers. JSPaint is + pinned at commit `53be67a`. + - **Labels:** `laya/paintdata.py` labels every state from what is on the canvas. It gives the action to take + there and the three judgement answers. + - **Behaviour:** the labeller drives, with random actions mixed in (probability 0–0.3) and sometimes a random + start. + - **DAgger (`--model `):** a trained checkpoint drives each step with probability `1 - beta` instead, so + the data covers the states the model itself reaches, including its mistakes. + - **Output:** each run writes a new dataset under `/data/vqa/`; existing datasets are never replaced. +2. **`finetune_long`**, with `--head-max-len 1536 --max-len 2560`. The loader now takes `head_max_len` from the + checkpoint, so training builds the same uncut inputs as `predict`, and the value is saved with the new checkpoint. + Batch 16 fits on an A100 40 GB; batch 32 ran out of memory. +3. **`paint_eval`** plays a checkpoint in JSPaint on an L4, alongside the labeller, the older scripted expert and + random play. It saves each episode's final frame and GIF under `/paint_eval//`. + +### The labeller draws a target the model can see + +The first labeller always drew one circle, centred on the canvas. Nothing on screen showed where that circle +should start, so the model trained on it (`paint-circle-v1`) walked straight past the start and never knew when to +press. + +The labeller now draws the way a person does. It presses where the cursor is, if a circle fits there, and draws +clockwise from the circle's top. The rest of the target then follows from the drawing: the circle has a fixed +radius (0.18 of the canvas side) and its top is where the drawing began. As a policy, this labeller completes and +ends 6 of 6 circles by itself, scoring 0.92–0.94. + +### Results + +These are 5 episodes on unseen seeds (900000–900004) with `paint_eval`, so read them as a spot check. The row +data is in `docs/paint-circle-metrics.json`. + +| Checkpoint | Trained on | Mean score | Passed | Steps | +|---|---|---|---|---| +| `paint-circle-v1` | `paint_circle`: 240 episodes, fixed hidden target | 0.00 | 0/5 | 260 | +| `paint-circle-v2` | `paint_circle_v2`: 240 episodes, visible target, 2 passes | 0.00 | 0/5 | 260 | +| `paint-circle-v3` | v2's data plus `paint_circle_dagger1`: 120 episodes with v2 driving half the steps | 0.61 | 2/5 | 105 | +| Labeller (reference) | – | 0.94 | 5/5 | 98 | +| Random (reference) | – | 0.00 | 0/5 | 41 | + +- **v2** presses at once and draws the first half of the circle correctly: E, then SE, then S, then SW. It then + fails to keep turning at the bottom, drifts off the ring and runs into the corner. Its own judgement notices: + P(on track) falls from 1.0 to 0.01 as it leaves the ring. The labeller's data had almost no such states. +- **v3**, after one DAgger round, draws one continuous closed stroke in every episode, releases the button and + answers `DONE` by itself. It judges the result "circle" with progress 3.7–4.0 out of 4. The three misses fail + only on roundness (0.46–0.64; passing needs 0.7): they close up but are lopsided, with a small gap where the + stroke started. + +Each round took about 1.5 hours on Modal: data generation, about an hour of A100 training, then evaluation. diff --git a/tests/test_circle_verifier.py b/tests/test_circle_verifier.py new file mode 100644 index 00000000..faeb26de --- /dev/null +++ b/tests/test_circle_verifier.py @@ -0,0 +1,77 @@ +"""The circle verifier passes round circles (including the 16-sided walk the environment's fine moves draw) and fails +the shapes a policy might draw instead, including an octagon (what coarse eight-direction moves draw).""" +import math +import random + +import numpy as np +import pytest + +from laya.circle_verifier import fit_circle, score_circle + +PIL = pytest.importorskip("PIL") +from PIL import Image, ImageDraw # noqa: E402 + +W, H = 683, 384 # JSPaint's default canvas + + +def canvas(draw): + im = Image.new("RGB", (W, H), "white") + draw(ImageDraw.Draw(im)) + return np.asarray(im) + + +def polygon(n, r=110, cx=340, cy=190, rot=math.pi / 8): + return [(cx + r * math.cos(rot + 2 * math.pi * i / n), cy + r * math.sin(rot + 2 * math.pi * i / n)) + for i in range(n + 1)] + + +def test_fit_circle_recovers_centre_and_radius(): + t = np.linspace(0, 2 * np.pi, 200) + cx, cy, r = fit_circle(300 + 80 * np.cos(t), 150 + 80 * np.sin(t)) + assert (round(cx), round(cy), round(r)) == (300, 150, 80) + + +@pytest.mark.parametrize("name,draw", [ + ("ring", lambda d: d.ellipse([230, 80, 450, 300], outline="black", width=4)), + ("16-gon", lambda d: d.line(polygon(16), fill="black", width=4)), + ("32-gon", lambda d: d.line(polygon(32), fill="black", width=4)), + ("small ring", lambda d: d.ellipse([305, 155, 375, 225], outline="black", width=4)), + ("coloured ring", lambda d: d.ellipse([100, 50, 300, 250], outline=(200, 0, 0), width=6)), +]) +def test_circles_pass(name, draw): + s = score_circle(canvas(draw)) + assert s["passed"], (name, s) + assert s["score"] >= 0.85 and s["coverage"] == 1.0 + + +@pytest.mark.parametrize("name,draw", [ + ("blank", lambda d: None), + ("line", lambda d: d.line([100, 100, 500, 300], fill="black", width=4)), + ("dot", lambda d: d.ellipse([300, 150, 310, 160], fill="black")), + ("tiny ring", lambda d: d.ellipse([300, 150, 330, 180], outline="black", width=4)), + ("half arc", lambda d: d.arc([230, 80, 450, 300], 0, 180, fill="black", width=4)), + ("three-quarter arc", lambda d: d.arc([230, 80, 450, 300], 0, 270, fill="black", width=4)), + ("octagon", lambda d: d.line(polygon(8), fill="black", width=4)), + ("square", lambda d: d.rectangle([230, 80, 450, 300], outline="black", width=4)), + ("flat ellipse", lambda d: d.ellipse([140, 120, 540, 270], outline="black", width=4)), + ("filled disc", lambda d: d.ellipse([230, 80, 450, 300], fill="black")), + ("ring plus stray line", lambda d: (d.ellipse([230, 80, 450, 300], outline="black", width=4), + d.line([0, 20, 680, 20], fill="black", width=4))), +]) +def test_non_circles_fail(name, draw): + s = score_circle(canvas(draw)) + assert not s["passed"], (name, s) + assert s["score"] < 0.7 + + +def test_random_scribbles_fail(): + for seed in range(5): + rng = random.Random(seed) + pts = [(rng.uniform(0, W), rng.uniform(0, H)) for _ in range(15)] + assert not score_circle(canvas(lambda d: d.line(pts, fill="black", width=4)))["passed"] + + +def test_arc_scores_below_closed_circle(): + closed = score_circle(canvas(lambda d: d.ellipse([230, 80, 450, 300], outline="black", width=4))) + arc = score_circle(canvas(lambda d: d.arc([230, 80, 450, 300], 0, 270, fill="black", width=4))) + assert arc["score"] < closed["score"] and arc["max_gap_deg"] > 30 diff --git a/tests/test_paintdata.py b/tests/test_paintdata.py new file mode 100644 index 00000000..095b7a29 --- /dev/null +++ b/tests/test_paintdata.py @@ -0,0 +1,89 @@ +"""The JSPaint labeller's pieces on synthetic canvases: ring coverage, judgement labels and soft direction targets.""" +import math + +import numpy as np +import pytest + +from laya.paintdata import (DIRECTION_SPREAD, MARGIN, SECTORS, direction_target, fits, implied_circle, + label_judgements, radius, ring_state, start_point) + +PIL = pytest.importorskip("PIL") +from PIL import Image, ImageDraw # noqa: E402 + +S = 512 +R = radius(S) + + +def canvas(draw): + im = Image.new("RGB", (S, S), "white") + draw(ImageDraw.Draw(im)) + return np.asarray(im) + + +def box(cx, cy, r=R): + return [cx - r, cy - r, cx + r, cy + r] + + +def test_blank_canvas(): + px = canvas(lambda d: None) + assert implied_circle(px) is None + assert label_judgements(px) == {"progress": "0", "on_track": "on track", "drawn": "nothing"} + + +@pytest.mark.parametrize("cx,cy", [(256, 256), (150, 330), (380, 140)]) +def test_full_ring_anywhere_is_a_complete_circle(cx, cy): + px = canvas(lambda d: d.ellipse(box(cx, cy), outline="black", width=4)) + c = implied_circle(px) + assert abs(c[0] - cx) <= 2 and abs(c[1] - cy) <= 3 # centre one radius below the topmost ink + assert ring_state(px, c)["covered"].all() + assert label_judgements(px) == {"progress": "4", "on_track": "on track", "drawn": "circle"} + + +@pytest.mark.parametrize("extent,level", [(60, "1"), (200, "2"), (300, "3")]) +def test_arc_drawn_clockwise_from_the_top(extent, level): + # PIL angles run clockwise from 3 o'clock, so the top is 270 + px = canvas(lambda d: d.arc(box(256, 256), 270, 270 + extent, fill="black", width=4)) + lab = label_judgements(px) + assert lab == {"progress": level, "on_track": "on track", "drawn": "arc"} + + +def test_stray_ink_is_off_track_and_unlabelled_shape(): + def draw(d): + d.arc(box(256, 256), 270, 360, fill="black", width=4) + d.line([60, 420, 470, 470], fill="black", width=4) # far from the ring + lab = label_judgements(canvas(draw)) + assert lab["on_track"] == "off track" and lab["drawn"] is None + + +def test_wrong_size_ring_is_off_track(): + px = canvas(lambda d: d.ellipse(box(256, 256, 40), outline="black", width=4)) + assert label_judgements(px)["on_track"] == "off track" + + +def test_arc_whose_circle_cannot_fit_is_off_track(): + px = canvas(lambda d: d.arc(box(256, S - 40), 270, 300, fill="black", width=4)) # top near the bottom edge + assert not fits(implied_circle(px), S) + assert label_judgements(px)["on_track"] == "off track" + + +def test_start_point_keeps_the_circle_on_the_canvas(): + assert start_point((256, 100), S) == (256, 100) + x, y = start_point((500, 500), S) + assert fits((x, y + R, R), S) and y + 2 * R <= S - MARGIN + + +def test_direction_target_spreads_to_neighbours(): + from laya.games import paint_question + + options = list(paint_question()["action"]["criteria"]) + t = direction_target(options, "N") + assert math.isclose(sum(t), 1.0) + assert t[options.index("N")] == pytest.approx(1 - 2 * DIRECTION_SPREAD) + assert t[options.index("NbE")] == pytest.approx(DIRECTION_SPREAD) + assert t[options.index("NbW")] == pytest.approx(DIRECTION_SPREAD) # wraps around past N + pen = direction_target(options, "PEN_DOWN") + assert pen[options.index("PEN_DOWN")] == 1.0 and sum(pen) == 1.0 + + +def test_sector_count(): + assert SECTORS == 36 diff --git a/tests/test_paintenv.py b/tests/test_paintenv.py new file mode 100644 index 00000000..4edd03be --- /dev/null +++ b/tests/test_paintenv.py @@ -0,0 +1,149 @@ +"""The JSPaint environment end to end in headless Chromium: mouse actions reach the real canvas, and the scripted +circle expert passes the verifier. Skipped without Playwright, a Chromium, or a JSPaint checkout (JSPAINT_DIR, default +a sibling of this repo).""" +import os + +import numpy as np +import pytest + +pytest.importorskip("playwright") +pytest.importorskip("PIL") + +from laya.circle_verifier import ink_mask # noqa: E402 +from laya.games import paint_question # noqa: E402 +from laya.paintenv import ACTIONS, TOOLS, compass_moves, JSPaintEnv, JSPaintServer, circle_expert, play_episodes, \ + random_policy # noqa: E402 + +JSPAINT = os.environ.get("JSPAINT_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "jspaint")) +if not os.path.exists(os.path.join(JSPAINT, "index.html")): + pytest.skip("no JSPaint checkout at %s" % JSPAINT, allow_module_level=True) + + +@pytest.fixture(scope="module") +def env(): + server = JSPaintServer(JSPAINT) + try: + e = JSPaintEnv(server.url) + except Exception as err: # no Chromium installed + server.close() + pytest.skip("cannot launch Chromium: %s" % err) + yield e + e.close() + server.close() + + +def test_question_covers_actions_and_tools_are_mouse_only(): + assert set(paint_question()["action"]["criteria"]) == set(ACTIONS) + assert len(ACTIONS) == 32 + 3 + assert set(paint_question(directions=8)["action"]["criteria"]) == set(compass_moves(8)) | {"PEN_DOWN", "PEN_UP", + "DONE"} + assert set(compass_moves(16)) < set(compass_moves(32)) + for ux, uy in compass_moves(32).values(): + assert abs(ux * ux + uy * uy - 1) < 1e-5 + assert {t["name"] for t in TOOLS} == {"move_mouse", "mouse_down", "mouse_up", "screenshot"} + + +def test_state_has_two_frames_and_recent_actions(env): + env.reset(seed=2) + first = env.state() + assert set(first) == {"images", "context"} + assert len(first["images"]) == 2 and first["images"][0] is first["images"][1] + assert "none yet" in first["context"] and "pen is up" in first["context"] + acts = ["PEN_DOWN"] + ["E"] * (env.frame_gap + 3) + for a in acts: + env.step(a) + st = env.state() + old, now = st["images"] + assert old is not now and old.size == now.size == (env.width, env.height) + assert ("oldest first: %s." % " ".join(acts[-env.history:])) in st["context"] + assert "pen is down" in st["context"] + + +def test_full_question_fits_raised_budget_without_truncation(): + """The full 35-option question plus a two-frame state must fit PAINT_HEAD_MAX_LEN / PAINT_MAX_LEN uncut.""" + from transformers import AutoProcessor + from laya.paintenv import PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN + + try: + tok = AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM-256M-Instruct").tokenizer + except Exception as err: # offline + pytest.skip("tokenizer unavailable: %s" % err) + q = paint_question()["action"] + opts = ["- %s: %s" % kv for kv in q["criteria"].items()] + n_opts = sum(len(tok(o)["input_ids"]) + 1 for o in opts) + n_ins = len(tok(q["instructions"])["input_ids"]) + assert max(len(tok(o)["input_ids"]) for o in opts) <= 48 # the builder's per-option cap + assert n_opts + n_ins + 32 <= PAINT_HEAD_MAX_LEN + assert n_opts + n_ins + 2 * 64 + 256 <= PAINT_MAX_LEN # two 512px images are 64 tokens each + + +def test_judgement_questions_and_summary(): + from laya.games import PAINT_SHAPES, paint_judgements + from laya.paintenv import summarize_judgements + + q = paint_judgements("circle") + assert set(q) == {"progress", "on_track", "drawn"} + assert q["progress"]["type"] == "score" and len(q["progress"]["criteria"]) == 5 + assert set(q["on_track"]["criteria"]) == {"on track", "off track"} + assert set(q["drawn"]["criteria"]) == set(PAINT_SHAPES) and "circle" in PAINT_SHAPES and "oval" in PAINT_SHAPES + fake = {"progress": {"score": 2.5, "probabilities": {str(i): 0.2 for i in range(5)}}, + "on_track": {"probabilities": {"on track": 0.7, "off track": 0.3}}, + "drawn": {"choice": "arc", "probabilities": {"arc": 0.6}}} + j = summarize_judgements(fake) + assert j["progress"] == 2.5 and j["on_track"] == 0.7 and j["drawn"] == "arc" + + +def test_reset_is_blank_and_observation_is_canvas(env): + obs = env.reset(seed=3) + assert obs.size == (env.width, env.height) == (512, 512) + assert not ink_mask(env.canvas_pixels()).any() + assert not env.done and not env.pen + + +def test_pen_down_moves_draw_and_pen_up_moves_do_not(env): + env.reset(seed=0) + x0, y0 = env.cursor + env.step("PEN_DOWN") + for _ in range(4): + env.step("E") + env.step("PEN_UP") + for _ in range(3): + env.step("S") + ink = ink_mask(env.canvas_pixels()) + ys, xs = np.nonzero(ink) + assert len(xs) > 50 + assert abs(np.median(ys) - y0) <= 3 # one horizontal stroke on the starting row + assert xs.min() >= x0 - 5 and xs.max() <= x0 + 4 * env.step_px + 5 + assert env.cursor == (x0 + 4 * env.step_px, y0 + 3 * env.step_px) + + +def test_cursor_is_clamped_to_canvas(env): + env.reset(seed=0) + for _ in range(200): + env.step("NW") + assert env.cursor == (1.0, 1.0) + + +def test_done_ends_episode_with_verifier_result(env): + env.reset(seed=0) + _, reward, done, info = env.step("DONE") + assert done and reward == 0.0 and info["verifier"]["score"] == 0.0 + with pytest.raises(RuntimeError): + env.step("N") + + +def test_call_tool_draws(env): + env.reset(seed=1) + env.call_tool("mouse_down") + env.call_tool("move_mouse", {"dx": 60, "dy": 30}) + env.call_tool("mouse_up") + assert ink_mask(env.canvas_pixels()).sum() > 50 + assert env.call_tool("screenshot").size == (env.width, env.height) + + +def test_expert_passes_and_random_does_not(env): + expert = play_episodes(env, circle_expert(), episodes=2, seed=0) + assert expert["pass_rate"] == 1.0 and expert["mean_score"] >= 0.9 + assert all(e["roundness"] >= 0.9 for e in expert["results"]) + rnd = play_episodes(env, random_policy(0), episodes=3, seed=0) + assert rnd["pass_rate"] == 0.0 and rnd["mean_score"] < expert["mean_score"] diff --git a/tests/test_quickdraw.py b/tests/test_quickdraw.py new file mode 100644 index 00000000..4fa0c829 --- /dev/null +++ b/tests/test_quickdraw.py @@ -0,0 +1,107 @@ +"""Quick, Draw! doodles as drawing tasks, offline: stroke placement, resampling, rasterising, and the doodle +labeller's judgement labels (the browser and the network are not needed).""" +import math + +import numpy as np +import pytest + +from laya.quickdraw import (BITMAP, DOODLE_FRAC, DoodleLabeller, _resample, doodle_strokes, drawn_options, + render_strokes, to_bitmap) + +PIL = pytest.importorskip("PIL") +from PIL import Image, ImageDraw # noqa: E402 + +S = 512 +HOUSE = {"key_id": "test", "strokes": [[(0, 120), (0, 255), (200, 255), (200, 120), (0, 120)], + [(0, 120), (100, 0), (200, 120)]]} + + +class FakeEnv: + def __init__(self, cursor=(256.0, 256.0), step=6): + self.canvas_size, self.cursor, self.step_px, self.pen = S, cursor, step, False + from laya.paintenv import compass_moves + + self.moves = compass_moves(32) + + +def test_resample_spacing_and_ends(): + pts = _resample([(0, 0), (30, 0), (30, 12)], 6) + assert pts[0] == (0.0, 0.0) and pts[-1] == (30.0, 12.0) + gaps = [math.dist(a, b) for a, b in zip(pts, pts[1:])] + assert max(gaps) <= 6 + 1e-6 + + +def test_doodle_starts_at_cursor_and_fits(): + strokes = doodle_strokes(HOUSE["strokes"], S, (200.0, 150.0), 6) + assert strokes[0][0] == pytest.approx((200.0, 150.0)) + pts = np.array([p for s in strokes for p in s]) + assert (pts.max(0) - pts.min(0)).max() == pytest.approx(DOODLE_FRAC * S, rel=0.02) + near_edge = doodle_strokes(HOUSE["strokes"], S, (500.0, 500.0), 6) # would run off: shifted to fit + pts = np.array([p for s in near_edge for p in s]) + assert pts.min() >= 8 - 1e-6 and pts.max() <= S - 8 + 1e-6 + + +def test_bitmap_is_cropped_and_normalised(): + bm = to_bitmap(render_strokes(HOUSE["strokes"])) + assert bm.shape == (BITMAP, BITMAP) and bm.max() <= 1.0 and bm.sum() > 20 + assert to_bitmap(np.zeros((64, 64), bool)).sum() == 0 + + +def draw_strokes(strokes, upto=None): + im = Image.new("RGB", (S, S), "white") + d = ImageDraw.Draw(im) + for s in strokes[:upto]: + d.line(s, fill="black", width=4) + return np.asarray(im) + + +def test_labeller_starts_by_pressing_at_the_cursor_and_labels_judgements(): + env = FakeEnv() + lab = DoodleLabeller("house", HOUSE, drawn_options()) + lab.reset(env) + assert lab.action(env, draw_strokes(lab.strokes, 0)) == "PEN_DOWN" + blank = lab.judgements(draw_strokes(lab.strokes, 0)) + assert blank == {"progress": "0", "on_track": "on track", "drawn": "nothing"} + lab.i, lab.j = len(lab.strokes), 0 # as if every stroke were drawn + done = lab.judgements(draw_strokes(lab.strokes)) + assert done == {"progress": "4", "on_track": "on track", "drawn": "house"} + assert lab.action(env, draw_strokes(lab.strokes)) == "DONE" + + +def test_stray_ink_is_off_track(): + env = FakeEnv() + lab = DoodleLabeller("house", HOUSE, drawn_options()) + lab.reset(env) + lab.i = 1 + px = draw_strokes(lab.strokes, 1) + im = Image.fromarray(px.copy()) + ImageDraw.Draw(im).line([(10, 500), (500, 480), (20, 470), (490, 460)], fill="black", width=4) + assert lab.judgements(np.asarray(im))["on_track"] == "off track" + + +def test_drawn_options_cover_training_categories_and_nothing(): + opts = drawn_options(("house", "apple")) + assert list(opts) == ["house", "apple", "nothing"] and opts["apple"].startswith("a doodle of an apple") + + +def test_grouped_choice_moves_when_moving_is_likelier_than_any_pen_action(): + from laya.paintenv import COMPASS, grouped_choice + + probs = {d: 0.02 for d in COMPASS} # moving: 0.66 in total, spread thin + probs.update(E=0.04, PEN_DOWN=0.0, PEN_UP=0.30, DONE=0.02) + assert max(probs, key=probs.get) == "PEN_UP" and grouped_choice(probs) == "E" + probs.update(PEN_UP=0.9) + assert grouped_choice(probs) == "PEN_UP" + + +def test_sampled_choice_follows_the_grouped_probabilities(): + import collections + import random + + from laya.paintenv import COMPASS, sampled_choice + + probs = {d: 0.0 for d in COMPASS} + probs.update(E=0.3, N=0.3, PEN_DOWN=0.0, PEN_UP=0.4, DONE=0.0) + rng = random.Random(0) + c = collections.Counter(sampled_choice(probs, rng) for _ in range(4000)) + assert set(c) == {"E", "N", "PEN_UP"} and abs(c["PEN_UP"] / 4000 - 0.4) < 0.03