Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
d4b1e80
Add a JSPaint mouse-only environment and a circle verifier
claude Sep 26, 2026
02c9e26
JSPaint env: 16-direction 8 px moves and a roundness-based circle ver…
claude Sep 26, 2026
8b1e259
JSPaint env: 32-direction 6 px moves and a cleaner expert start
claude Sep 26, 2026
aeafd6a
Docs: zero-shot model row for the 32-direction JSPaint defaults
claude Sep 26, 2026
56e9d50
JSPaint env: square canvas the size of laya-vision's input (512x512)
claude Sep 26, 2026
1d37095
Docs: zero-shot model row on the 512x512 JSPaint canvas
claude Sep 26, 2026
b31dea7
JSPaint env: two-frame memory, recent actions, and the full question …
claude Sep 26, 2026
0bad323
JSPaint env: judgement questions asked with each action
claude Sep 26, 2026
cb1b150
JSPaint training data and Modal jobs: state-based labeller, prepare_p…
claude Sep 27, 2026
f149fc6
modal_app: install git in the JSPaint image before cloning
claude Sep 27, 2026
78af9df
prepare_paint: collect shard results before zipping
claude Sep 27, 2026
8347cc8
JSPaint labeller: a target the model can see
claude Sep 27, 2026
4a893a6
JSPaint data: DAgger rounds with a trained checkpoint driving
claude Sep 27, 2026
4c53b0f
paint_eval: save each episode's final frame and GIF
claude Sep 27, 2026
eb24fd4
Docs: JSPaint training on Modal and the v1-v3 results
claude Sep 27, 2026
345209d
JSPaint: a square task (goal text only) and paint_eval --task
claude Sep 27, 2026
5f60be2
Quick, Draw! doodles as JSPaint drawing tasks
claude Sep 27, 2026
f8f5d89
prepare_paint: independent shard calls, retried waits, --resume
claude Sep 27, 2026
fb239a7
modal_app: put _get_retrying above prepare_paint's decorator
claude Sep 27, 2026
2f0eda3
ModelPolicy: choose move vs pen action first, then the direction
claude Sep 27, 2026
84fed8c
paintenv: rewrap ModelPolicy docstring
claude Sep 27, 2026
e4a2e1e
paintenv: rewrap ModelPolicy docstring
claude Sep 27, 2026
bf350c1
ModelPolicy: decide = argmax (default) | grouped | sample
claude Sep 27, 2026
b5523fb
modal_app: prompt_sensitivity, does the drawing policy read the task?
claude Sep 27, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 141 additions & 0 deletions docs/paint-circle-metrics.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
{
"task": "JSPaint circle, mouse-only, 512x512 canvas, 32 directions x 6 px",
"eval": "modal_app.py::paint_eval, 5 episodes, seeds 900000-900004, verifier laya/circle_verifier.py (pass >= 0.7)",
"runs": {
"paint-circle-v1/best": {
"data": "paint_circle (fixed hidden target, superseded)",
"policies": {
"model": {
"mean_score": 0.0,
"pass_rate": 0.0,
"mean_steps": 260.0,
"scores": [
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"expert": {
"mean_score": 0.9669000000000001,
"pass_rate": 1.0,
"mean_steps": 180.8,
"scores": [
0.9652,
0.9718,
0.9662,
0.9655,
0.9658
]
},
"random": {
"mean_score": 0.0,
"pass_rate": 0.0,
"mean_steps": 40.6,
"scores": [
0.0,
0.0,
0.0,
0.0,
0.0
]
}
}
},
"paint-circle-v2/best": {
"data": "paint_circle_v2",
"policies": {
"model": {
"mean_score": 0.0,
"pass_rate": 0.0,
"mean_steps": 260.0,
"scores": [
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"labeller": {
"mean_score": 0.9355399999999999,
"pass_rate": 1.0,
"mean_steps": 98.2,
"scores": [
0.9414,
0.9365,
0.9323,
0.9266,
0.9409
]
},
"expert": {
"mean_score": 0.9669000000000001,
"pass_rate": 1.0,
"mean_steps": 180.8,
"scores": [
0.9652,
0.9718,
0.9662,
0.9655,
0.9658
]
},
"random": {
"mean_score": 0.0,
"pass_rate": 0.0,
"mean_steps": 40.6,
"scores": [
0.0,
0.0,
0.0,
0.0,
0.0
]
}
}
},
"paint-circle-v3/best": {
"data": "paint_circle_v2 + paint_circle_dagger1 (v2 driving, beta 0.5)",
"policies": {
"model": {
"mean_score": 0.6131800000000001,
"pass_rate": 0.4,
"mean_steps": 105.2,
"scores": [
0.4635,
0.4997,
0.6361,
0.7043,
0.7623
]
},
"labeller": {
"mean_score": 0.9355399999999999,
"pass_rate": 1.0,
"mean_steps": 98.2,
"scores": [
0.9414,
0.9365,
0.9323,
0.9266,
0.9409
]
},
"random": {
"mean_score": 0.0,
"pass_rate": 0.0,
"mean_steps": 40.6,
"scores": [
0.0,
0.0,
0.0,
0.0,
0.0
]
}
}
}
}
}
132 changes: 132 additions & 0 deletions examples/jspaint_circle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
"""Ask Laya Vision to draw a circle in JSPaint with the mouse only, and score it.

Each step the model sees two screenshots of the JSPaint canvas (8 steps ago and now, the cursor marked on both) plus
its pen state and last 12 actions as text, and a `choice` question over mouse actions (a 6 px move toward each of 32
compass directions, pen down, pen up, done) picks the next move, which is sent to the real app as pointer events.
The model is loaded with raised token budgets and asked with strict=True, so the full question is never cut. At the
end the circle verifier scores the true canvas pixels. The scripted expert and a random policy play the same seeds as
reference points, and the model's score is also reported normalized, (model - random) / (expert - random), as in the
games suite.

pip install -e . playwright pillow
git clone https://github.com/r33drichards/jspaint ../jspaint
python examples/jspaint_circle.py --jspaint ../jspaint --policies expert,random # no model needed
python examples/jspaint_circle.py --jspaint ../jspaint --episodes 5 # + thaitea/laya-vision
python examples/jspaint_circle.py --model checkpoints/my-run --revision <sha> --device cuda

Chromium: set LAYA_CHROMIUM (or --chromium) to an existing binary instead of running `playwright install`.
Each run writes a new directory under --out (create-only): summary.json, and per policy and episode a
trajectory.jsonl (action, cursor, pen, and the model's action probabilities), final.png and episode.gif.
"""
import argparse
import json
import os
import time

from PIL import Image

from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, circle_expert, judge_canvas,
model_policy, play_episodes, random_policy)


def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--jspaint", default=os.path.join(os.path.dirname(__file__), "..", "..", "jspaint"),
help="path to a JSPaint checkout (default: a sibling of this repo)")
ap.add_argument("--policies", default="model,expert,random", help="comma list of model, expert, random")
ap.add_argument("--episodes", type=int, default=5)
ap.add_argument("--seed", type=int, default=900_000)
ap.add_argument("--max-steps", type=int, default=260)
ap.add_argument("--step-px", type=int, default=6)
ap.add_argument("--directions", type=int, default=32, choices=(8, 16, 32))
ap.add_argument("--model", default="thaitea/laya-vision")
ap.add_argument("--revision", default=None, help="pin the Hub revision of --model")
ap.add_argument("--device", default=None)
ap.add_argument("--chromium", default=None, help="Chromium executable (default: LAYA_CHROMIUM or Playwright's)")
ap.add_argument("--canvas-size", type=int, default=None,
help="square canvas side (default: the model's input image_size, 512 for the released checkpoints)")
ap.add_argument("--no-judge", action="store_true",
help="skip the judgement questions (progress, on track, what is drawn) the model answers each step")
ap.add_argument("--judge-stop", action="store_true",
help="let the model's own judgement end the episode: DONE once it rates the task complete")
ap.add_argument("--headed", action="store_true", help="show the browser window")
ap.add_argument("--out", default="results/jspaint")
args = ap.parse_args()

out_dir = os.path.join(args.out, time.strftime("%Y%m%d-%H%M%S"))
os.makedirs(out_dir, exist_ok=False)
names = [p.strip() for p in args.policies.split(",") if p.strip()]
summary = {"task": "circle", "jspaint": os.path.abspath(args.jspaint), "args": vars(args), "policies": {}}

agent = None
if "model" in names:
from laya import load_vlm

# raised token budgets: the full question and a two-frame state fit with nothing cut
agent = load_vlm(args.model, revision=args.revision, device=args.device, head_max_len=PAINT_HEAD_MAX_LEN,
max_len=PAINT_MAX_LEN)
# A square canvas the size of the model's input, so the screenshot reaches the vision tower unscaled.
canvas_size = args.canvas_size or (agent.prep.image_size if agent is not None else 512)
summary["canvas_size"] = canvas_size

with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, max_steps=args.max_steps,
step_px=args.step_px, directions=args.directions,
canvas_size=canvas_size, headless=not args.headed,
executable_path=args.chromium, keep_frames=True) as env:
for name in names:
if name == "model":
policy = model_policy(agent, judge=not args.no_judge, judge_stop=args.judge_stop)
elif name == "expert":
policy = circle_expert()
elif name == "random":
policy = random_policy(args.seed)
else:
raise SystemExit("unknown policy %r" % name)

rows, final_labels = [], {}

def on_step(env, action):
row = dict(env.trajectory[-1], episode=env.seed)
if name == "model":
row["probabilities"] = policy.last["probabilities"]
row["act_probability"] = policy.last["action"]["act_probability"]
if policy.last_judgement:
row["judgement"] = policy.last_judgement
rows.append(row)
if env.done:
if agent is not None and not args.no_judge:
# what the clean final canvas (no cursor) actually shows, for hindsight relabelling
final_labels[env.seed] = judge_canvas(agent, Image.fromarray(env.canvas_pixels()))
ep_dir = os.path.join(out_dir, name, "seed%d" % env.seed)
os.makedirs(ep_dir)
with open(os.path.join(ep_dir, "trajectory.jsonl"), "w") as f:
f.writelines(json.dumps(r) + "\n" for r in rows)
env.frames[-1].save(os.path.join(ep_dir, "final.png"))
env.frames[0].save(os.path.join(ep_dir, "episode.gif"), save_all=True,
append_images=env.frames[1:], duration=120, loop=0)
rows.clear()

t = time.time()
res = play_episodes(env, policy, args.episodes, seed=args.seed, on_step=on_step)
res["seconds"] = round(time.time() - t, 1)
for ep in res["results"]:
if ep["seed"] in final_labels:
ep["judged_final"] = final_labels[ep["seed"]]
if name == "model":
res["model"], res["provenance"] = args.model, policy.provenance
summary["policies"][name] = res
print("%-7s mean score %.3f pass rate %.2f mean steps %.1f (%.0fs)" % (
name, res["mean_score"], res["pass_rate"], res["mean_steps"], res["seconds"]))

pol = summary["policies"]
if {"model", "expert", "random"} <= set(pol):
e, r = pol["expert"]["mean_score"], pol["random"]["mean_score"]
summary["normalized_model_score"] = (pol["model"]["mean_score"] - r) / (e - r) if e > r else None
print("model normalized score (model - random) / (expert - random): %s" % summary["normalized_model_score"])
with open(os.path.join(out_dir, "summary.json"), "w") as f:
json.dump(summary, f, indent=2, default=str)
print("wrote %s" % out_dir)


if __name__ == "__main__":
main()
113 changes: 113 additions & 0 deletions examples/jspaint_judge_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
"""Check whether Laya Vision's judgements track what is really happening on the JSPaint canvas.

The state-based labeller (``laya.paintdata.LabellerPolicy``) and a random policy play; every --every steps the
model answers the judgement questions (`laya.games.paint_judgements`: progress 0-4, on track or off track, what is
drawn) about the same two-frame state the drawing policy sees. The truth for progress is the share of the 36
angular sectors of the circle the drawing implies (``laya.paintdata.implied_circle``) that hold ink (0 before
drawing, 1 for a full ring). Reported:

* how judged progress rises with true progress along the labeller's circles (Spearman rank correlation, and the mean
judged progress per quarter of true progress);
* judged progress and P(on track) for the random policy, which should stay low;
* what the model says the clean final canvas shows, per policy.

python examples/jspaint_judge_check.py --jspaint ../jspaint --episodes 3

Writes a new directory under --out: judgements.jsonl (one row per check) and summary.json.
"""
import argparse
import json
import os
import time

import numpy as np
from PIL import Image

from laya.games import paint_judgements
from laya.paintdata import LabellerPolicy, implied_circle, ring_state
from laya.paintenv import (PAINT_HEAD_MAX_LEN, PAINT_MAX_LEN, JSPaintEnv, JSPaintServer, judge_canvas, random_policy,
summarize_judgements)


def true_progress(env) -> float:
"""Share of the sectors of the circle the drawing implies that hold ink (0 with nothing drawn)."""
px = env.visible_pixels()
circle = implied_circle(px, None, env.canvas_size)
return 0.0 if circle is None else ring_state(px, circle)["coverage"]


def spearman(a, b) -> float:
ra, rb = np.argsort(np.argsort(a)), np.argsort(np.argsort(b))
return float(np.corrcoef(ra, rb)[0, 1]) if len(a) > 2 and np.std(ra) and np.std(rb) else float("nan")


def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--jspaint", default=os.path.join(os.path.dirname(__file__), "..", "..", "jspaint"))
ap.add_argument("--episodes", type=int, default=3)
ap.add_argument("--every", type=int, default=10, help="ask the judgements every N steps")
ap.add_argument("--seed", type=int, default=900_000)
ap.add_argument("--model", default="thaitea/laya-vision")
ap.add_argument("--revision", default=None)
ap.add_argument("--device", default=None)
ap.add_argument("--chromium", default=None)
ap.add_argument("--out", default="results/jspaint-judge")
args = ap.parse_args()

from laya import load_vlm

out_dir = os.path.join(args.out, time.strftime("%Y%m%d-%H%M%S"))
os.makedirs(out_dir, exist_ok=False)
agent = load_vlm(args.model, revision=args.revision, device=args.device, head_max_len=PAINT_HEAD_MAX_LEN,
max_len=PAINT_MAX_LEN)
questions = paint_judgements("circle")
rows, finals = [], {}
with JSPaintServer(args.jspaint) as server, JSPaintEnv(server.url, canvas_size=agent.prep.image_size,
executable_path=args.chromium) as env:
for name, make in (("expert", LabellerPolicy), ("random", lambda: random_policy(args.seed))):
policy = make()
finals[name] = []
for i in range(args.episodes):
env.reset(args.seed + i)
while not env.done:
if env.steps % args.every == 0:
ans = agent.predict(env.state(), questions, strict=True)["answers"]
rows.append({"policy": name, "seed": args.seed + i, "step": env.steps, "pen": env.pen,
"true_progress": round(true_progress(env), 4), **summarize_judgements(ans)})
env.step(policy(env))
final = judge_canvas(agent, Image.fromarray(env.canvas_pixels()))
finals[name].append({"seed": args.seed + i, "verifier": env.result["score"], **final})
rows.append({"policy": name, "seed": args.seed + i, "step": env.steps, "final": True,
"true_progress": round(true_progress(env), 4), **final})
print("%-6s seed %d verifier %.2f final canvas judged: %s (progress %.2f, on track %.2f)" % (
name, args.seed + i, env.result["score"], final["drawn"], final["progress"], final["on_track"]))

exp = [r for r in rows if r["policy"] == "expert" and not r.get("final")]
rnd = [r for r in rows if r["policy"] == "random"]
tp, jp = [r["true_progress"] for r in exp], [r["progress"] for r in exp]
quarters = {}
for lo, hi in ((0, 0.001), (0.001, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.01)):
sel = [r for r in exp if lo <= r["true_progress"] < hi]
if sel:
quarters["%.2f-%.2f" % (lo, min(hi, 1.0))] = {
"n": len(sel), "judged_progress": round(float(np.mean([r["progress"] for r in sel])), 3),
"p_on_track": round(float(np.mean([r["on_track"] for r in sel])), 3)}
summary = {
"model": args.model, "provenance_revision": getattr(agent, "source", {}).get("revision"),
"every": args.every, "episodes": args.episodes, "seed": args.seed,
"expert_spearman_true_vs_judged_progress": round(spearman(tp, jp), 3),
"expert_by_true_progress": quarters,
"random_mean_judged_progress": round(float(np.mean([r["progress"] for r in rnd])), 3),
"random_mean_p_on_track": round(float(np.mean([r["on_track"] for r in rnd])), 3),
"final_canvas": finals,
}
with open(os.path.join(out_dir, "judgements.jsonl"), "w") as f:
f.writelines(json.dumps(r) + "\n" for r in rows)
with open(os.path.join(out_dir, "summary.json"), "w") as f:
json.dump(summary, f, indent=2)
print(json.dumps({k: v for k, v in summary.items() if k != "final_canvas"}, indent=2))
print("wrote %s" % out_dir)


if __name__ == "__main__":
main()
Loading
Loading