diff --git a/docs/specs/homes.md b/docs/specs/homes.md index f6cbe0e..15fa61d 100644 --- a/docs/specs/homes.md +++ b/docs/specs/homes.md @@ -36,7 +36,7 @@ LAYERFS_HOME (default ~/.layerfs) LayerFS's bucket-.key pg---.pw pg-----.pw job/ (0700) storage jobs about this home (§5) - storage-jobs.json push, pull, move, backup and restore jobs + storage-jobs.json push, pull, move, backup, restore and compact jobs doctor-jobs.json home check reports stat/ (0700) storage statistics about this home (§5) storage-observations/ Store statistics journals diff --git a/src/client/storage/HomeOverview.tsx b/src/client/storage/HomeOverview.tsx index 311502c..f8d909b 100644 --- a/src/client/storage/HomeOverview.tsx +++ b/src/client/storage/HomeOverview.tsx @@ -184,7 +184,7 @@ export function HomeOverview(props: HomeOverviewProps): ReactNode { if (!j.error && j.outcome !== 'failed' && j.outcome !== 'interrupted' && j.outcome !== 'unknown') continue needs.push({ key: `job:${j.id}`, tone: 'danger', rank: 0, - title: {endedWords(j, t)}, + title: {endedWords(j, t, bytes)}, detail: j.error ? <>{j.error.message} · {ago(j.ended, now, t as Translate)} : ago(j.ended, now, t as Translate), action: { label: t('jobRetry'), run: () => props.onRetry(j) }, extra: , diff --git a/src/client/storage/JobProgress.tsx b/src/client/storage/JobProgress.tsx index 088b081..9ff2be2 100644 --- a/src/client/storage/JobProgress.tsx +++ b/src/client/storage/JobProgress.tsx @@ -17,7 +17,7 @@ export function JobPanel({ job: j, t, onRetry, onDismiss }: { job: StorageJob, t
- {running ? doingWords(j, t) : endedWords(j, t)} + {running ? doingWords(j, t) : endedWords(j, t, bytes)} {running ? {t('jobStep', { at, of: steps.length, step: stepWords(j.phase, t) })} : null} {running && j.total ? {t('jobBytes', { done: bytes(j.done ?? 0), total: bytes(j.total) })} : null} @@ -52,7 +52,7 @@ export function JobList({ jobs, t, now }: { jobs: readonly StorageJob[], t: Stor
  • - {j.ended ? endedWords(j, t) : doingWords(j, t)} + {j.ended ? endedWords(j, t, bytes) : doingWords(j, t)} {kindWords(j.kind, t)} {j.target} · {ago(j.ended ?? j.started, now, t as Translate)}
  • diff --git a/src/client/storage/StoragePage.tsx b/src/client/storage/StoragePage.tsx index 7aa4a4c..f4de423 100644 --- a/src/client/storage/StoragePage.tsx +++ b/src/client/storage/StoragePage.tsx @@ -150,6 +150,8 @@ function StorageIde({ api, doctorT, t, openRef, map, more }: { api: LayerfsApi, if (j.kind === 'pull') return api.call('POST', `/api/stores/${enc(j.store)}/pull`) if (j.kind === 'move') return api.call('POST', `/api/stores/${enc(j.store)}/move`, { machine: j.target }) if (j.kind === 'backup') return api.call('POST', `/api/stores/${enc(j.store)}/backup`, { bucket: j.target, ...(j.file ? { file: j.file } : {}) }) + // a new run: it settles what the failed one left before it marks again + if (j.kind === 'compact') return api.call('POST', `/api/stores/${enc(j.store)}/compact`) return api.call('POST', `/api/buckets/${enc(j.target)}/restore`, { store: j.store, ...(j.file ? { file: j.file } : {}) }) } const openMachine = (name: string) => open(`machine:${name}`) @@ -162,6 +164,13 @@ function StorageIde({ api, doctorT, t, openRef, map, more }: { api: LayerfsApi, backup: () => setDialog(view && view.buckets.length === 0 ? { kind: 'bucket', then: { kind: 'backup', store: s, known: [] } } : { kind: 'backup', store: s }), backupTo: (l) => act(() => api.call('POST', `/api/stores/${enc(s.name)}/backup`, { bucket: l.bucket, file: l.file })), restore: (bucket, file) => setDialog({ kind: 'restore', bucket, file, store: s.name }), + // only a runtime that has store.compact gets the menu row + ...(view?.compact ? { + compact: () => setDialog({ + kind: 'confirm', title: t('confirmTitleCompact', { name: s.name }), body: t('confirmCompactBody'), action: t('compactAction'), plain: true, + run: () => api.call('POST', `/api/stores/${enc(s.name)}/compact`).then((job) => { if (job && typeof job === 'object' && 'id' in job && 'kind' in job) take([job as StorageJob]) }), + }), + } : {}), remove: () => setDialog({ kind: 'confirm', title: t('confirmTitleRemove', { name: s.name }), body: t('confirmRemoveBody', { n: s.projects.length }), action: t('removeAction'), name: s.name, run: () => api.call('DELETE', `/api/stores/${enc(s.name)}?confirm=${enc(s.name)}`).then(() => closeStore(s.name)), diff --git a/src/client/storage/StoreCard.tsx b/src/client/storage/StoreCard.tsx index 676197f..0e23512 100644 --- a/src/client/storage/StoreCard.tsx +++ b/src/client/storage/StoreCard.tsx @@ -8,7 +8,7 @@ import { ago, bytes } from '../files/format.ts' import type { Translate } from '../files/types.ts' import { LinkButton, MoreMenu } from '../ui/index.tsx' import type { StorageT } from './locale.ts' -import { inHome, latestBackup } from './model.ts' +import { compactBlock, inHome, latestBackup } from './model.ts' import type { StorageJob, StoreBackupLink, StoreRow } from './types.ts' import css from './Storage.module.css' @@ -18,6 +18,8 @@ export interface StoreActions { move: () => void backup: () => void backupTo: (link: StoreBackupLink) => void + /** Confirm, then compact the Store; absent when the selected LayerFS runtime has no store.compact. */ + compact?: (() => void) | undefined restore: (bucket: string, file: string) => void remove: () => void disconnect: () => void @@ -52,14 +54,17 @@ export function Location({ store: s, t, onMachine }: { store: StoreRow, t: Stora */ export function StoreMenu({ store: s, on, t, machine }: { store: StoreRow, on: StoreActions, t: StorageT, machine?: string | undefined }): ReactNode { const pg = s.kind === 'postgresql' + const blocked = compactBlock(s, t) return ( {(close) => { const run = (fn: () => void) => () => { close(); fn() } + const compact = on.compact ? {t('compactStore')} : null return pg ? ( <> on.editConnection())}>{t('msEditConnection')} on.check(machine))}>{t('pgCheckFrom')} + {compact} {t('disconnect')} ) : ( @@ -78,6 +83,7 @@ export function StoreMenu({ store: s, on, t, machine }: { store: StoreRow, on: S {`${t('backupNow')} · ${l.bucket}/${l.file}`} ))} + {compact} {t('remove')} ) diff --git a/src/client/storage/dialogs.tsx b/src/client/storage/dialogs.tsx index 0ba2cbb..4d75394 100644 --- a/src/client/storage/dialogs.tsx +++ b/src/client/storage/dialogs.tsx @@ -50,7 +50,8 @@ export type StorageDialog = | { kind: 'restore', bucket: string, file: string, store?: string | null } /** `then`: the dialog Add a bucket returns to, closed or saved. */ | { kind: 'bucket', then?: StorageDialog } - | { kind: 'confirm', title: string, body: string, action: string, name?: string, force?: ConfirmForce, run: (force: boolean) => Promise } + /** `plain`: the action removes nothing, so its button is not the danger one (Compact). */ + | { kind: 'confirm', title: string, body: string, action: string, name?: string, force?: ConfirmForce, plain?: boolean, run: (force: boolean) => Promise } /** An optional Force checkbox on a confirm: checked, the action runs with force and reads `action`. */ export interface ConfirmForce { label: string, action: string } @@ -84,7 +85,7 @@ export function StorageDialogs({ api, t, dialog, view, onClose, done, onDialog } case 'backups': return case 'restore': return case 'bucket': return onDialog(dialog.then!) : onClose} done={done} /> - case 'confirm': return after(dialog.run(force))} /> + case 'confirm': return after(dialog.run(force))} /> } } @@ -545,14 +546,14 @@ function BucketDialog({ api, t, onClose, done }: { api: LayerfsApi, t: StorageT, // ---------- Confirm ---------- /** A destructive confirmation; with `name`, the name is typed to confirm. */ -function ConfirmDialog({ t, title, body, action, name, force, onClose, run }: { - t: StorageT, title: string, body: string, action: string, name?: string | undefined, force?: ConfirmForce | undefined, onClose: () => void, run: (force: boolean) => Promise +function ConfirmDialog({ t, title, body, action, name, force, plain, onClose, run }: { + t: StorageT, title: string, body: string, action: string, name?: string | undefined, force?: ConfirmForce | undefined, plain?: boolean | undefined, onClose: () => void, run: (force: boolean) => Promise }): ReactNode { const [typed, setTyped] = useState('') const [forced, setForced] = useState(false) const on = !!force && forced return ( - run(on)}> + run(on)}> {name ? : null} {force ? : null} {!name && !force ? : null} diff --git a/src/client/storage/locale.ts b/src/client/storage/locale.ts index 7a95242..fc01357 100644 --- a/src/client/storage/locale.ts +++ b/src/client/storage/locale.ts @@ -148,17 +148,25 @@ const en = { stepUpload: 'uploading', stepDownload: 'downloading', stepHash: 'hashing', + stepMark: 'marking', + stepBuild: 'building', + stepSwap: 'swapping', + stepCleanup: 'cleaning up', stepStarting: 'starting', jobPushing: 'Pushing to {place}', jobPulling: 'Pulling from {place}', jobMoving: 'Moving to {place}', jobBackingUp: 'Backing up to {place}', jobRestoring: 'Restoring from {place}', + jobCompacting: 'Compacting {place}', jobPushed: 'Pushed to {place}', jobPulled: 'Pulled from {place}', jobMoved: 'Moved to {place}', jobBackedUp: 'Backed up to {place}', jobRestored: 'Restored from {place}', + jobCompacted: 'Compacted {place}', + jobCompactedFreed: 'Compacted {place} · freed {freed}', + jobCompactedNone: 'Compacted {place} · nothing to reclaim', jobFailed: '{what} {place} failed in {phase}', jobUnknown: 'Outcome unknown · inspect the Store', jobInterrupted: 'Interrupted · final result unavailable', @@ -169,6 +177,7 @@ const en = { kindMove: 'Move to', kindBackup: 'Backup to', kindRestore: 'Restore from', + kindCompact: 'Compaction of', jobRetry: 'Retry', recentOperations: 'Recent operations', recentOperationsNone: 'No push, pull, move, backup or restore in this session.', @@ -492,6 +501,12 @@ const en = { connectBucket: 'Connect bucket', confirmTitleRemove: 'Remove {name}?', confirmRemoveBody: 'The Store and every project in it ({n}) are deleted with all their history. Backups in buckets stay.', + confirmTitleCompact: 'Compact {name}?', + confirmCompactBody: 'Reclaims the space of removed projects, Branches and files by rebuilding the Store. The Store refuses other work until it ends.', + compactAction: 'Compact', + compactStore: 'Compact…', + compactNeedsHome: 'Pull the Store to LayerFS Home first', + compactNeedsIdle: 'The Store is not idle', confirmTitlePg: 'Disconnect the PostgreSQL Store {name}?', confirmPgBody: 'This home forgets the Store and its projects. The database and its data stay; connect it again to use them.', confirmTitleBucket: 'Disconnect the bucket {name}?', @@ -1073,17 +1088,25 @@ const zh: typeof en = { stepUpload: '上传中', stepDownload: '下载中', stepHash: '计算哈希', + stepMark: '标记中', + stepBuild: '重建中', + stepSwap: '切换中', + stepCleanup: '清理中', stepStarting: '启动中', jobPushing: '正在推送到 {place}', jobPulling: '正在从 {place} 拉回', jobMoving: '正在迁移到 {place}', jobBackingUp: '正在备份到 {place}', jobRestoring: '正在从 {place} 恢复', + jobCompacting: '正在压缩 {place}', jobPushed: '已推送到 {place}', jobPulled: '已从 {place} 拉回', jobMoved: '已迁移到 {place}', jobBackedUp: '已备份到 {place}', jobRestored: '已从 {place} 恢复', + jobCompacted: '已压缩 {place}', + jobCompactedFreed: '已压缩 {place} · 释放 {freed}', + jobCompactedNone: '已压缩 {place} · 没有可回收的空间', jobFailed: '{what} {place} 在 {phase} 阶段失败', jobUnknown: '结果未知 · 请检查 Store', jobInterrupted: '已中断 · 最终结果不可用', @@ -1094,6 +1117,7 @@ const zh: typeof en = { kindMove: '迁移到', kindBackup: '备份到', kindRestore: '恢复自', + kindCompact: '压缩', jobRetry: '重试', recentOperations: '最近的操作', recentOperationsNone: '本次会话中没有推送、拉回、迁移、备份或恢复。', @@ -1410,6 +1434,12 @@ const zh: typeof en = { connectBucket: '连接存储桶', confirmTitleRemove: '移除 {name}?', confirmRemoveBody: 'Store 及其中所有项目({n} 个)将连同全部历史一起删除。存储桶中的备份保留。', + confirmTitleCompact: '压缩 {name}?', + confirmCompactBody: '通过重建 Store 回收已移除的项目、分支和文件所占的空间。结束前 Store 会拒绝其他操作。', + compactAction: '压缩', + compactStore: '压缩…', + compactNeedsHome: '请先将 Store 拉回 LayerFS Home', + compactNeedsIdle: 'Store 当前不空闲', confirmTitlePg: '断开 PostgreSQL Store {name}?', confirmPgBody: '此 Home 将忘记该 Store 及其项目。数据库及数据保留;重新连接即可再次使用。', confirmTitleBucket: '断开存储桶 {name}?', diff --git a/src/client/storage/model.ts b/src/client/storage/model.ts index 53d5385..1b78f9a 100644 --- a/src/client/storage/model.ts +++ b/src/client/storage/model.ts @@ -6,6 +6,17 @@ import type { StorageT } from './locale.ts' /** A sqlite Store saved under the LayerFS home (`native`). */ export const inHome = (s: StoreRow): boolean => s.kind === 'sqlite' && s.machine === 'native' +/** + * Why store.compact cannot start on a Store now, or null. LayerFS compacts a + * sqlite Store only where its file is, and the home is the one place this app + * runs it; a PostgreSQL Store is compacted on its server. + */ +export function compactBlock(s: StoreRow, t: StorageT): string | null { + if (s.kind === 'sqlite' && !inHome(s)) return t('compactNeedsHome') + if (s.state !== 'idle') return t('compactNeedsIdle') + return null +} + /** project.init goes into a Store on any machine, not while it is down or busy. */ export const canAddProject = (s: StoreRow): boolean => s.kind === 'postgresql' || (s.state !== 'down' && s.state !== 'busy') @@ -159,6 +170,8 @@ export const STEPS: Record = { move: ['check', 'snapshot', 'send', 'verify', 'install', 'record', 'retire'], backup: ['check', 'snapshot', 'send', 'check', 'hash', 'upload'], restore: ['check', 'download', 'verify', 'install'], + // a sqlite Store's; the host names a PostgreSQL Store's own (`steps`) + compact: ['mark', 'build', 'verify', 'install'], } export type StepState = 'done' | 'now' | 'failed' | 'todo' @@ -166,7 +179,9 @@ export type StepState = 'done' | 'now' | 'failed' | 'todo' /** One run of a call's steps with each step's state. */ export function stepsOf(j: StorageJob): { step: string, state: StepState }[] { const seen = j.phases?.filter((p) => p.half === (j.half ?? null)).map((p) => p.phase) - const steps = seen?.length ? seen : [...STEPS[j.kind], ...(!STEPS[j.kind].includes(j.phase ?? '') && j.phase ? [j.phase] : [])] + // a job that names its steps keeps them in that order from the start, plus any phase it did not name + const named = j.steps?.length ? [...j.steps, ...(seen ?? []).filter((p) => !j.steps!.includes(p))] : null + const steps = named ?? (seen?.length ? seen : [...STEPS[j.kind], ...(!STEPS[j.kind].includes(j.phase ?? '') && j.phase ? [j.phase] : [])]) const ok = j.ended !== null && j.error === null const at = j.error ? steps.lastIndexOf(j.error.phase ?? '') : j.phase ? steps.lastIndexOf(j.phase) : j.ended ? -1 : 0 return steps.map((step, i) => ({ step, state: ok || i < at ? 'done' : i === at ? (j.error ? 'failed' : 'now') : 'todo' })) @@ -196,6 +211,10 @@ export function stepWords(step: string | null, t: StorageT): string { case 'upload': return t('stepUpload') case 'download': return t('stepDownload') case 'hash': return t('stepHash') + case 'mark': return t('stepMark') + case 'build': return t('stepBuild') + case 'swap': return t('stepSwap') + case 'cleanup': return t('stepCleanup') case null: return t('stepStarting') default: return step } @@ -209,15 +228,16 @@ export function doingWords(j: StorageJob, t: StorageT): string { case 'move': return t('jobMoving', { place: placeWords(j.target, t) }) case 'backup': return t('jobBackingUp', { place: j.target }) case 'restore': return t('jobRestoring', { place: j.file ? `${j.target}/${j.file}` : j.target }) + case 'compact': return t('jobCompacting', { place: j.store }) } } -/** How a job ended, in words. */ -export function endedWords(j: StorageJob, t: StorageT): string { +/** How a job ended, in words. `size` writes a byte count (a compaction's freed space); without it the count is left out. */ +export function endedWords(j: StorageJob, t: StorageT, size?: (n: number) => string): string { if (j.summary) return j.summary if (j.outcome === 'unknown') return t('jobUnknown') if (j.outcome === 'interrupted') return t('jobInterrupted') - const place = j.kind === 'pull' ? placeWords(j.from, t) : j.kind === 'backup' || j.kind === 'restore' ? j.target : placeWords(j.target, t) + const place = j.kind === 'pull' ? placeWords(j.from, t) : j.kind === 'backup' || j.kind === 'restore' ? j.target : j.kind === 'compact' ? j.store : placeWords(j.target, t) if (j.error) return t('jobFailed', { what: kindWords(j.kind, t), place, phase: j.error.phase ?? '—' }) switch (j.kind) { case 'push': return t('jobPushed', { place }) @@ -225,6 +245,12 @@ export function endedWords(j: StorageJob, t: StorageT): string { case 'move': return t('jobMoved', { place }) case 'backup': return t('jobBackedUp', { place }) case 'restore': return t('jobRestored', { place }) + case 'compact': { + // the report's own words: nothing was worth rebuilding, or what the Store shrank by + if (j.result?.path === 'none') return t('jobCompactedNone', { place }) + const freed = j.result?.physical_shrink + return size && typeof freed === 'number' && freed > 0 ? t('jobCompactedFreed', { place, freed: size(freed) }) : t('jobCompacted', { place }) + } } } @@ -236,6 +262,7 @@ export function kindWords(kind: JobKind, t: StorageT): string { case 'move': return t('kindMove') case 'backup': return t('kindBackup') case 'restore': return t('kindRestore') + case 'compact': return t('kindCompact') } } diff --git a/src/client/storage/types.ts b/src/client/storage/types.ts index a9e3bdf..f324a95 100644 --- a/src/client/storage/types.ts +++ b/src/client/storage/types.ts @@ -92,9 +92,9 @@ export interface BucketRow { stores: string[] } -export type JobKind = 'push' | 'pull' | 'move' | 'backup' | 'restore' +export type JobKind = 'push' | 'pull' | 'move' | 'backup' | 'restore' | 'compact' -/** A push, pull, move, backup or restore the host started. */ +/** A push, pull, move, backup, restore or compaction the host started. */ export interface StorageJob { id: string kind: JobKind @@ -111,6 +111,10 @@ export interface StorageJob { error: (SdkRefusal & { phase: string | null, note?: 'pulled' }) | null outcome?: 'succeeded' | 'failed' | 'interrupted' | 'unknown' | null phases?: { phase: string, half: string | null, at: number }[] + /** The call's phases in order, when the host knows them before the run (a compaction). */ + steps?: readonly string[] + /** A finished compaction's report (store.compact): the path it took and what the Store shrank by. */ + result?: { path?: string | null, physical_shrink?: number | null } | null summary?: string | null } @@ -124,6 +128,8 @@ export interface StorageView { stores: StoreRow[] buckets: BucketRow[] jobs: StorageJob[] + /** Whether the selected LayerFS runtime has store.compact; absent from an older host. */ + compact?: boolean at: number } diff --git a/src/host/routes/index.ts b/src/host/routes/index.ts index 1d206f9..c6e0179 100644 --- a/src/host/routes/index.ts +++ b/src/host/routes/index.ts @@ -27,7 +27,7 @@ * Events, published with `ctx.layerfs.emit(type, data)`; `data` is Connect's * whole HostEvent object (its `type` included), so a client handler written * for Connect's event stream takes `event.data` unchanged: - * storage-job { type: 'storage-job', job: StorageJob } a push/pull/move/backup/restore job moved + * storage-job { type: 'storage-job', job: StorageJob } a push/pull/move/backup/restore/compact job moved * storage { type: 'storage' } the Storage view changed (read it again) * doctor { type: 'doctor', id, state } a Doctor job changed * demo-preset { jobId, preset, state } a preset build or reset moved diff --git a/src/host/routes/jobs.ts b/src/host/routes/jobs.ts index 9c2480a..b5aee43 100644 --- a/src/host/routes/jobs.ts +++ b/src/host/routes/jobs.ts @@ -10,7 +10,7 @@ import { stringify } from './util.ts' /** One storage job (Connect's StorageJob). */ export interface StorageJob { id: string - kind: 'push' | 'pull' | 'move' | 'backup' | 'restore' + kind: 'push' | 'pull' | 'move' | 'backup' | 'restore' | 'compact' store: string target: string from: string | null @@ -20,6 +20,8 @@ export interface StorageJob { half: 'pull' | 'push' | null phase: string | null phases: { phase: string, half: string | null, at: number }[] + /** The phases the call reports, in order, when the host knows them before the run (a compaction). */ + steps?: readonly string[] outcome: 'succeeded' | 'failed' | 'unknown' | 'interrupted' | null pid: number | null observation: any @@ -33,6 +35,9 @@ export interface StorageJob { } /** Transfer failures retain the CLI's declared authority. Never infer rollback. */ +/** The phases store.compact reports, by Store family (LayerFS `on_progress`, GC decision A13). */ +export const COMPACT_STEPS = { sqlite: ['mark', 'build', 'verify', 'install'], postgresql: ['mark', 'build', 'swap', 'cleanup'] } as const + export function failureOutcome(error: any, phase: string | null): StorageJob['outcome'] { if (error?.code === 'Unknown' || (error?.transport && error.transport !== 'refusal') || ['install', 'record'].includes(phase ?? '')) return 'unknown' if (error?.code === 'Interrupted') return 'interrupted' @@ -55,6 +60,8 @@ export async function inspectJob(lfs: Lfs, job: StorageJob) { const at = Date.now() if (job.kind === 'backup') return { at, kind: 'backup', value: await lfs.backup.list(job.target), action: null } if (job.kind === 'restore') return { at, kind: 'restore', value: await lfs.store.get(job.store), action: null } + // a compaction is never resolved from here: the next store.compact judges what a run left + if (job.kind === 'compact') return { at, kind: 'compact', value: await lfs.store.status(job.store), action: null } const status = await lfs.store.status(job.store) const direction = job.kind === 'move' ? job.half ?? 'pull' : job.kind const expected = `store.${direction}` diff --git a/src/host/routes/router.ts b/src/host/routes/router.ts index bcdfc9e..294ce59 100644 --- a/src/host/routes/router.ts +++ b/src/host/routes/router.ts @@ -183,6 +183,8 @@ export async function route(g: Groups, method: string, parts: string[], query: U if (id && sub === 'move' && method === 'POST') return storage.move(id, body?.machine) // { bucket, file? } if (id && sub === 'backup' && method === 'POST') return storage.backup(id, body ?? {}) + // reclaim the Store's dead bytes (store.compact), as a job + if (id && sub === 'compact' && method === 'POST') return storage.compact(id) return undefined } if (group === 'buckets') { diff --git a/src/host/routes/storage.ts b/src/host/routes/storage.ts index 3c31fdf..459fa2e 100644 --- a/src/host/routes/storage.ts +++ b/src/host/routes/storage.ts @@ -27,6 +27,7 @@ * (#537 U5); a push that fails leaves the Store on native * place the one of push, pull or move that puts a Store on a machine * backup/restore backup.create / backup.restore as jobs + * compact store.compact as a job, its phases from on_progress * * Buckets and PostgreSQL Stores take env:/file: references, which LayerFS * resolves; a key or password entered in the panel is written by the host to @@ -44,7 +45,7 @@ import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync import { basename, dirname, join } from 'node:path' import { execute } from '../layerfs/cli.ts' import { BackupMarks, historyFingerprint, remoteOnly, settleRemote } from './backup-heads.ts' -import { JobStore, failureOutcome, inspectJob, transferSummary, type StorageJob } from './jobs.ts' +import { COMPACT_STEPS, JobStore, failureOutcome, inspectJob, transferSummary, type StorageJob } from './jobs.ts' import { computer } from './computer.ts' import { cliFeatures, refuse, wire, type RouteHost } from './util.ts' import { leftoverOf, walksOf } from './observed.ts' @@ -374,6 +375,8 @@ export class Storage { stores, buckets, jobs, + // whether the selected runtime has store.compact (older ones do not) + compact: (await this.compactCall()) !== null, at: Date.now(), } // a write while this read ran: answer it, but do not keep it for the join @@ -686,9 +689,9 @@ export class Storage { * `run(on_progress, step, on_spawn)` may make more than one call: `step(half)` * marks the half a move is in. */ - start(kind: StorageJob['kind'], store: string, target: string, file: string | null | undefined, run: (on_progress: (p: any) => void, step: (half: 'pull' | 'push', from?: string | null) => void, on_spawn: (pid: number) => void) => Promise, from: string | null = null) { + start(kind: StorageJob['kind'], store: string, target: string, file: string | null | undefined, run: (on_progress: (p: any) => void, step: (half: 'pull' | 'push', from?: string | null) => void, on_spawn: (pid: number) => void) => Promise, from: string | null = null, steps?: readonly string[]) { if ([...this.jobs.values()].some((j) => j.store === store && !j.ended)) throw refuse(409, 'ConnectBusy', `this host is already running a job on ${store}; wait for it to end`) - const job: StorageJob = { id: randomUUID(), kind, store, target, from, file: file ?? null, started: Date.now(), ended: null, half: null, phase: null, phases: [], outcome: null, pid: null, observation: null, summary: null, done: null, total: null, error: null, result: null } + const job: StorageJob = { id: randomUUID(), kind, store, target, from, file: file ?? null, started: Date.now(), ended: null, half: null, phase: null, phases: [], outcome: null, pid: null, observation: null, summary: null, done: null, total: null, error: null, result: null, ...(steps ? { steps } : {}) } this.jobs.set(job.id, job) while (this.jobs.size > KEEP_JOBS) { const oldest = [...this.jobs.values()].find((j) => j.ended) @@ -724,7 +727,8 @@ export class Storage { (error: any) => { // the SDK's refusal as it came: its code and text, the phase it stopped in, and its detail (entity, op, related) job.phase = error?.detail?.phase ?? job.phase - job.outcome = failureOutcome(error, job.phase) + // a transfer that stops in install or record is unknown; a compaction's refusal says so itself (`Unknown`) + job.outcome = failureOutcome(error, kind === 'compact' ? null : job.phase) job.error = { transport: error?.transport ?? null, code: error?.code ?? 'Failed', message: error?.message ?? String(error), phase: error?.detail?.phase ?? job.phase, detail: error?.detail ?? null, ...(error?.note ? { note: error.note } : {}) } this.log(`[storage] ${kind} ${store}: ${error?.code ?? ''} ${error?.message ?? error}`) }, @@ -823,6 +827,28 @@ export class Storage { return this.start('restore', store, bucket, file, (on_progress, _step, on_spawn) => this.lfs.backup.restore(store, bucket, file || undefined, { on_progress, on_spawn })) } + /** The selected runtime's store.compact: null when it has none, and whether it reports phases. */ + private async compactCall(): Promise<{ progress: boolean } | null> { + const found = await this.host.discover().catch(() => null) + const entry = found?.calls.find((c) => c.family === 'store' && c.call === 'compact') + return entry ? { progress: entry.options.some((o) => o.name === 'on_progress') } : null + } + + /** + * store.compact(store): rebuild a Store from its live history, where it is + * (a sqlite file in the home, or its PostgreSQL server). LayerFS refuses a + * sqlite Store on another machine, a Store with mounts, and a busy one; the + * job carries that refusal. One attempt: nothing here runs it again. + */ + async compact(store: unknown) { + checkName(store, 'A compaction') + const call = await this.compactCall() + if (!call) throw refuse(409, 'Unsupported', 'The selected LayerFS runtime has no store.compact; choose a newer one in Set up LayerFS') + const family = (await this.lfs.store.get(store))?.kind === 'postgresql' ? 'postgresql' : 'sqlite' + // a runtime whose store.compact takes no on_progress runs without steps + return this.start('compact', store, store, null, (on_progress, _step, on_spawn) => this.lfs.store.compact(store, call.progress ? { on_progress, on_spawn } : { on_spawn }), null, COMPACT_STEPS[family]) + } + async inspect(id: string) { const job = this.jobs.get(id) if (!job || !job.ended) throw refuse(409, 'ConnectBusy', 'Wait for a completed job before inspecting recovery') diff --git a/test/call-table.test.mjs b/test/call-table.test.mjs index 4333277..caeadcb 100644 --- a/test/call-table.test.mjs +++ b/test/call-table.test.mjs @@ -236,6 +236,8 @@ const GATED = [ // `featuresOf(...).leftovers` reads the call table for it, checked below // the home family (layerfs-setup.md §3): the setup check uses home.get / home.init only when the call table has both (else one machine.list) { site: /^home\.(get|init)$/, file: 'src/host/service.ts', gate: /this\.homeCalls\(\)/, lines: 8 }, + // GC (layerfs-project#41): the route refuses `Unsupported` when the call table has no store.compact, and the view hides the menu row + { site: /^store\.compact$/, file: 'src/host/routes/storage.ts', gate: /const call = await this\.compactCall\(\)\n\s+if \(!call\) throw refuse\(409, 'Unsupported'/, lines: 6 }, // the preset Reset's force-close refuses up front (preflight) when machine.stop lacks the bool force option { site: /^machine\.stop --force$/, file: 'src/host/presets/machine-stop.js', gate: /option\.name === 'force' && option\.kind === 'bool'/, lines: 45 }, ] diff --git a/test/logs-classify.test.mjs b/test/logs-classify.test.mjs index 2039b20..e5b0985 100644 --- a/test/logs-classify.test.mjs +++ b/test/logs-classify.test.mjs @@ -11,13 +11,13 @@ import { join } from 'node:path' /** Calls of the pinned CLI that change state (or may: `workspace.exec` runs a command). */ const WRITES = new Set([ - 'project.init', 'project.export', 'project.remove', - 'branch.fork', 'branch.merge', 'branch.publish', + 'project.init', 'project.export', 'project.remove', 'project.delete', 'project.remove_file', + 'branch.fork', 'branch.merge', 'branch.publish', 'branch.remove', 'file.write', 'file.edit', 'file.rm', 'file.mkdir', 'file.mv', 'file.cp', 'file.apply', 'sandbox.create', 'sandbox.close', 'workspace.mount', 'workspace.exec', 'workspace.exec_cancel', 'workspace.commit', 'workspace.unmount', 'machine.start', 'machine.stop', 'machine.create', 'machine.remove', 'machine.update', 'machine.load', 'machine.prune', - 'store.create', 'store.connect', 'store.update', 'store.push', 'store.pull', 'store.remove', 'store.disconnect', + 'store.create', 'store.connect', 'store.update', 'store.push', 'store.pull', 'store.remove', 'store.compact', 'store.disconnect', 'backup.connect', 'backup.disconnect', 'backup.create', 'backup.restore', 'doctor.fix', ]) diff --git a/test/routes.test.mjs b/test/routes.test.mjs index f7cfc21..abec5fc 100644 --- a/test/routes.test.mjs +++ b/test/routes.test.mjs @@ -555,6 +555,67 @@ test('storage jobs: push runs as a job with storage-job events; one job per Stor } finally { rmSync(root, { recursive: true, force: true }) } }) +test('storage jobs: compact runs as a job with its steps, phases and report; an older runtime has none', async () => { + const compactCall = (options) => async () => ({ ...(await discovery()), calls: [{ family: 'store', call: 'compact', args: [{ name: 'name', kind: 'name' }], options }] }) + let finish, given + const impl = (kind) => ({ + ...base, + store: { + ...base.store, + get: async () => ({ name: 's1', kind, machine: kind === 'sqlite' ? 'native' : null }), + compact: (store, options) => new Promise((resolve) => { given = options; options.on_progress?.({ phase: 'mark', done: null, total: null }); options.on_progress?.({ phase: 'build', done: null, total: null }); finish = resolve }), + }, + }) + // a runtime whose store.compact declares on_progress + const { http, root } = fakeGroups(impl('sqlite'), compactCall([{ name: 'on_progress', kind: 'function' }])) + try { + assert.equal((await http('GET', '/api/storage')).body.compact, true) + const j = await http('POST', '/api/stores/s1/compact') + assert.equal(j.status, 200) + assert.equal(j.body.kind, 'compact') + assert.deepEqual(j.body.steps, ['mark', 'build', 'verify', 'install']) + await new Promise((r) => setTimeout(r, 10)) + assert.equal(typeof given.on_progress, 'function') + const running = await http('GET', `/api/storage-jobs/${j.body.id}`) + assert.equal(running.body.phase, 'build') + assert.deepEqual(running.body.phases.map((p) => p.phase), ['mark', 'build']) + assert.equal((await http('POST', '/api/stores/s1/compact')).body.code, 'ConnectBusy') + finish({ kind: 'Compacted', path: 'rebuild', physical_shrink: 4096 }) + await new Promise((r) => setTimeout(r, 10)) + const done = await http('GET', `/api/storage-jobs/${j.body.id}`) + assert.equal(done.body.outcome, 'succeeded') + assert.deepEqual(done.body.result, { kind: 'Compacted', path: 'rebuild', physical_shrink: 4096 }) + } finally { rmSync(root, { recursive: true, force: true }) } + // a PostgreSQL Store names its own steps; a refusal in `install` keeps the SDK's own code (not a transfer's unknown) + const pg = fakeGroups({ ...impl('postgresql'), store: { ...impl('postgresql').store, compact: async () => { throw new Refusal('Capacity', 'no space', { phase: 'install' }) } } }, compactCall([{ name: 'on_progress', kind: 'function' }])) + try { + const j = await pg.http('POST', '/api/stores/s1/compact') + assert.deepEqual(j.body.steps, ['mark', 'build', 'swap', 'cleanup']) + await new Promise((r) => setTimeout(r, 10)) + const failed = await pg.http('GET', `/api/storage-jobs/${j.body.id}`) + assert.equal(failed.body.outcome, 'failed') + assert.equal(failed.body.error.code, 'Capacity') + } finally { rmSync(pg.root, { recursive: true, force: true }) } + // store.compact without on_progress: the job runs, and the option is not passed + const plain = fakeGroups(impl('sqlite'), compactCall([])) + try { + await plain.http('POST', '/api/stores/s1/compact') + await new Promise((r) => setTimeout(r, 10)) + assert.equal(given.on_progress, undefined) + finish({ kind: 'Compacted', path: 'none' }) + await new Promise((r) => setTimeout(r, 10)) + } finally { rmSync(plain.root, { recursive: true, force: true }) } + // a runtime with no store.compact: the view says so and the route refuses before any call + const old = fakeGroups(impl('sqlite')) + try { + assert.equal((await old.http('GET', '/api/storage')).body.compact, false) + const refused = await old.http('POST', '/api/stores/s1/compact') + assert.equal(refused.status, 409) + assert.equal(refused.body.code, 'Unsupported') + assert.ok(!old.calls.includes('store.compact')) + } finally { rmSync(old.root, { recursive: true, force: true }) } +}) + test('project init job and saved reads route to their calls', async () => { const head = 'c'.repeat(66) const src = mkdtempSync(join(tmpdir(), 'dsh-routes-src-')) diff --git a/test/storage-jobs-model.test.mjs b/test/storage-jobs-model.test.mjs new file mode 100644 index 0000000..2f09db7 --- /dev/null +++ b/test/storage-jobs-model.test.mjs @@ -0,0 +1,55 @@ +// A storage job in words and steps (src/client/storage/model.ts): a compaction +// keeps the steps the host named, in order, from the first event. +// node --test test/storage-jobs-model.test.mjs +import { test, after } from 'node:test' +import assert from 'node:assert/strict' +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { build } from 'esbuild' + +const root = new URL('..', import.meta.url).pathname +mkdirSync(join(root, 'lib'), { recursive: true }) +const out = mkdtempSync(join(root, 'lib', '.test-storage-jobs-')) +after(() => rmSync(out, { recursive: true, force: true })) +await build({ absWorkingDir: root, stdin: { contents: "export * from './src/client/storage/model.ts'", resolveDir: root, loader: 'ts' }, outfile: join(out, 'model.mjs'), bundle: true, format: 'esm', platform: 'node', target: 'node24', logLevel: 'warning' }) +const M = await import(pathToFileURL(join(out, 'model.mjs')).href) + +const t = (key, vars = {}) => `${key}${Object.keys(vars).length ? ' ' + JSON.stringify(vars) : ''}` +const job = (over = {}) => ({ id: 'j', kind: 'compact', store: 's1', target: 's1', file: null, started: 1, ended: null, phase: null, done: null, total: null, error: null, phases: [], steps: ['mark', 'build', 'verify', 'install'], ...over }) +const store = (over = {}) => ({ name: 's1', kind: 'sqlite', machine: 'native', state: 'idle', ...over }) + +test('a compaction shows its named steps from the start and moves through them', () => { + assert.deepEqual(M.stepsOf(job()).map((s) => s.state), ['now', 'todo', 'todo', 'todo']) + const building = job({ phase: 'build', phases: [{ phase: 'mark', half: null, at: 1 }, { phase: 'build', half: null, at: 2 }] }) + assert.deepEqual(M.stepsOf(building).map((s) => `${s.step}:${s.state}`), ['mark:done', 'build:now', 'verify:todo', 'install:todo']) + assert.equal(M.shareOf(building), 0.25) + // a phase the host did not name is appended, not dropped + const extra = job({ phase: 'cleanup', phases: [{ phase: 'mark', half: null, at: 1 }, { phase: 'cleanup', half: null, at: 2 }] }) + assert.deepEqual(M.stepsOf(extra).map((s) => s.step), ['mark', 'build', 'verify', 'install', 'cleanup']) + // a failed run marks the step it stopped in + const failed = job({ ended: 9, phase: 'build', error: { code: 'Capacity', message: 'no space', phase: 'build' } }) + assert.deepEqual(M.stepsOf(failed).map((s) => s.state), ['done', 'failed', 'todo', 'todo']) + // a job without named steps keeps the phases it saw (push, pull, backup) + const push = job({ kind: 'push', steps: undefined, phase: 'send', phases: [{ phase: 'check', half: null, at: 1 }, { phase: 'send', half: null, at: 2 }] }) + assert.deepEqual(M.stepsOf(push).map((s) => s.step), ['check', 'send']) +}) + +test('a compaction ends in the words of its report', () => { + assert.equal(M.doingWords(job(), t), 'jobCompacting {"place":"s1"}') + assert.equal(M.endedWords(job({ ended: 9, result: { path: 'none' } }), t), 'jobCompactedNone {"place":"s1"}') + const size = (n) => `${n} B` + assert.equal(M.endedWords(job({ ended: 9, result: { path: 'rebuild', physical_shrink: 4096 } }), t, size), 'jobCompactedFreed {"place":"s1","freed":"4096 B"}') + // no shrink reported, or no way to write a size: the plain words + assert.equal(M.endedWords(job({ ended: 9, result: { path: 'rebuild', physical_shrink: 0 } }), t, size), 'jobCompacted {"place":"s1"}') + assert.equal(M.endedWords(job({ ended: 9, result: { path: 'rebuild', physical_shrink: 4096 } }), t), 'jobCompacted {"place":"s1"}') + assert.match(M.endedWords(job({ ended: 9, phase: 'build', error: { code: 'Capacity', message: 'no space', phase: 'build' } }), t), /^jobFailed .*"phase":"build"/) +}) + +test('compaction starts only where LayerFS runs it', () => { + assert.equal(M.compactBlock(store(), t), null) + assert.equal(M.compactBlock(store({ kind: 'postgresql', machine: null }), t), null) + assert.equal(M.compactBlock(store({ machine: 'lima1' }), t), 'compactNeedsHome') + assert.equal(M.compactBlock(store({ state: 'busy' }), t), 'compactNeedsIdle') + assert.equal(M.compactBlock(store({ kind: 'postgresql', machine: null, state: 'unresolved' }), t), 'compactNeedsIdle') +})