Add cleaning export/API streaming for faster collection access

README.md

@@ -229,6 +229,7 @@ the same version hash as the simple one.
 | `GET .../versions/negotiate/:sessionId`           | Session status, and the result or error of an async commit                                                  |
 | `GET .../versions/:semver/manifest`               | Version manifest (add `?since=` for delta; both keyset-paginated)                                           |
 | `GET .../versions/:semver/records`                | Paginated records                                                                                           |
+| `GET .../versions/:semver/records.ndjson`         | Every record streamed as NDJSON in one request — the bulk read path, resumable via `?after=`                |
 | `GET .../versions/:semver/diff?from=...`          | Diff between two versions                                                                                   |
 | `POST /api/records/batch`                         | Fetch records by hash (JSONL stream)                                                                        |
 | `GET /api/records/:hash/provenance`               | Find all collections containing a record                                                                    |

public/llms.txt

@@ -72,6 +72,7 @@ GET /api/collections/:owner/:slug/versions/latest       → latest version with
 GET /api/collections/:owner/:slug/versions/:semver      → specific version by semver (e.g. /versions/v1.2.0)
 
 ### Read records and files
+GET /api/collections/:owner/:slug/versions/:semver/records.ndjson → ALL records, streamed as NDJSON in one request (?type=TypeName&after=recordId)
 GET /api/collections/:owner/:slug/versions/:semver/records   → records for a version (?type=TypeName&limit=100&after=recordId)
 GET /api/collections/:owner/:slug/versions/:semver/manifest  → manifest: record ids/types/hashes + file hashes + schema hashes (?since=v1.0.0 for delta)
 GET /api/collections/:owner/:slug/versions/:semver/files     → list files for a version (hash, size, content type)
@@ -92,15 +93,21 @@ GET /api/collections/:owner/:slug/versions/:semver/diff?from=:semver → diff be
 GET /api/collections/:owner/:slug/export                → download .tar.gz archive (manifest.json + records/*.ndjson + files/*)
 GET /api/collections/:owner/:slug/export?version=v2.0.0  → export a specific version
 
-Export assembles the whole archive in memory, so it is only offered on collections below
-250,000 records. Above that it returns 413 with the record count and the limit. To read a
-large collection, page the records endpoint with ?after= (see Pagination below), or fetch
-the manifest if you only need id/type/hash — both work at any size.
+Archive layout: manifest.json, records/<Type>.ndjson per record type, and files/<hash>.
+Types too large for a single archive entry are split into numbered parts —
+records/<Type>.0000.ndjson, records/<Type>.0001.ndjson, ... — at 25,000 records per part.
+A type that fits in one part keeps the unnumbered records/<Type>.ndjson name, so archives
+of ordinary collections are unchanged. Read every records/*.ndjson entry and you have the
+version, whichever form it took.
 
-The same 250,000-record limit applies to the SQL explorer (/api/query/...), which builds an
-in-memory SQLite copy of a version. It is a UI feature rather than a documented API; on a
-large collection use the records endpoint, or Hot, which hydrates a collection into a
-queryable database built for the purpose.
+Export is capped at 2,000,000 records and returns 413 above that — a guard against handing
+back a multi-gigabyte tarball from a single GET, not a memory limit. For bulk reads prefer
+records.ndjson (below): it streams, resumes, and needs no unpacking.
+
+The SQL explorer (/api/query/...) has a lower limit of 250,000 records, because unlike
+export it genuinely does hold the whole version in memory to build a SQLite copy. It is a
+UI feature rather than a documented API; on a large collection use records.ndjson, or Hot,
+which hydrates a collection into a queryable database built for the purpose.
 
 ### Fork
 POST /api/collections/:owner/:slug/fork                 → fork collection into caller's org (requires write auth)
@@ -445,8 +452,12 @@ To paginate through all records (works at any collection size):
    GET .../records?limit=2000&after=<nextCursor>
 3. Repeat until hasMore is false.
 
-Ask for the largest page you can handle. Walking a whole collection is bounded by request
-count, not bytes — 60 requests/minute unauthenticated, 5,000 authenticated — so a
+BUT: if you want the whole collection, do not page it. Use the NDJSON stream below —
+paging costs one round trip per page purely to re-establish a cursor the server just had.
+Paging is for browsing a slice; streaming is for reading everything.
+
+Ask for the largest page you can handle. Walking a whole collection by paging is bounded by
+request count, not bytes — 60 requests/minute unauthenticated, 5,000 authenticated — so a
 3-million-record collection is 6,200 requests at 500/page and 1,550 at 2,000/page.
 Authenticate for any full-collection walk.
 
@@ -455,6 +466,62 @@ beyond that. Use ?after=<nextCursor> keyset pagination instead.
 
 ---
 
+## Bulk Read (the fast way to get a whole collection)
+
+GET /api/collections/:owner/:slug/versions/:semver/records.ndjson
+
+Streams every record in the version as newline-delimited JSON in a single response. One
+request regardless of size: a 3.1M-record collection is one call here versus 1,556 paged
+ones. The server reads through a database cursor and writes as it goes, so memory is
+constant on both ends — you can start processing the first line before the last is sent.
+
+Content-Type: application/x-ndjson
+X-Underlay-Record-Count: 3113504     ← how many lines to expect
+
+One JSON object per line:
+{"id":"arxiv:0704.0001","type":"Preprint","data":{...},"hash":"abc123..."}
+
+Parameters:
+- type:  restrict to one record type
+- after: resume — return records with ids strictly after this value
+
+Guarantees you can rely on:
+- Records are ordered by id, ascending. This is what makes `after` work.
+- `hash` is the same content-address the records endpoint serves: the full record hash for
+  owners, the public hash for everyone else.
+- Privacy filtering is identical to /records — private types and private records are
+  absent, private fields are stripped.
+
+Verify completeness yourself. A stream that dies halfway cannot report an error: the 200
+status and headers were already sent. Count the lines and compare against
+X-Underlay-Record-Count (or the version's recordCount). If they differ, resume with
+?after=<id of the last complete line you parsed> rather than starting over.
+
+That resume behaviour makes this strictly better than paging for bulk reads: the same
+recovery from a dropped connection, at a fraction of the requests.
+
+One edge: a record id is not guaranteed unique within a version — the same id can appear
+under more than one hash. `after` resumes strictly past the id, so if a stream broke
+between two lines sharing an id, resuming from it skips the second. Rare, and identical to
+how ?after= behaves on the paged endpoint, but if you need exactness compare the final line
+count against X-Underlay-Record-Count and re-read from an earlier id if it falls short.
+
+### Compression
+
+All /api/ responses are compressed when you send Accept-Encoding: gzip — about 3x on
+record data. Most HTTP clients do this automatically. It applies to the NDJSON stream too.
+
+### When to use which
+
+- Whole collection, one pass       → records.ndjson
+- A page, or browsing              → records?limit=2000&after=
+- Only ids/types/hashes            → manifest (≈120 bytes/record, far smaller than bodies)
+- What changed since a version     → manifest?since= (delta)
+- Specific records you know hashes for → POST /api/records/batch (up to 10,000 per call)
+- An archive to keep               → export (.tar.gz)
+
+---
+
 ## Record Format
 
 {

server.ts

@@ -7,6 +7,7 @@ import { serveStatic } from '@hono/node-server/serve-static'
 import { Scalar } from '@scalar/hono-api-reference'
 import { Hono } from 'hono'
 import { createOpenApiDocument } from 'hono-zod-openapi'
+import { compress } from 'hono/compress'
 import { cors } from 'hono/cors'
 import { marked } from 'marked'
 import type { ViteDevServer } from 'vite'
@@ -115,6 +116,14 @@ app.use(
 app.get('/agent/:token', agentHandlers.agentPage)
 
 // --- Auth + rate limiting for API routes ---
+// Responses are JSON and compress ~3x on real record data (measured on arXiv:
+// 3.08 MB -> 1.00 MB for a 2,000-record page). Nothing was compressing before —
+// not the app, and not Caddy, which has no `encode` directive — so every bulk
+// read was paying full size on the wire. Applied in the app rather than at the
+// proxy so local dev matches production, and because it covers the streaming
+// endpoints too (chunked transfer compresses fine).
+app.use('/api/*', compress())
+
 app.use('/api/*', authMiddleware)
 app.use('/api/*', rateLimitMiddleware)
 

src/api/collections.ts

@@ -14,10 +14,11 @@ import { getLatestReadyVersion, getOrgRole, hasOrgAccess } from '../lib/version-
 import { type AuthEnv } from './auth.server.js'
 import { requireAuth } from './auth.server.js'
 
-// Export builds the whole archive in memory (see the guard in the export route),
-// so it is offered only below this. Matches the SQL explorer's limit — both are
-// whole-collection-in-memory features and should draw the line in the same place.
-const MAX_EXPORT_RECORDS = 250_000
+// Export streams in bounded parts, so this is no longer a memory limit — it is a
+// guard against handing someone a multi-GB tarball from a single GET. Bulk reads
+// belong on records.ndjson. The SQL explorer keeps the lower 250k limit because
+// that one genuinely does hold the collection in memory.
+const MAX_EXPORT_RECORDS = 2_000_000
 
 const app = new Hono<AuthEnv>()
   // Browse collections — public by default, or the caller's own with ?mine=true
@@ -821,20 +822,20 @@ const app = new Hono<AuthEnv>()
         return c.json({ error: 'No versions found', statusCode: 404 }, 404)
       }
 
-      // The archive is assembled in memory: every record of a type is collected
-      // into a string[] and joined before it becomes a tar entry. That is fine
-      // for the collections this was built for and fatal on a multi-million
-      // record one — the join alone would exceed V8's maximum string length,
-      // and the array would exhaust the heap first, from an endpoint any
-      // visitor can reach. Refuse above the threshold rather than fall over.
+      // Export used to be capped because it assembled the whole archive in
+      // memory. It now emits bounded parts, so size is no longer the limit —
+      // but it is still a single long-running response that reads every record,
+      // and `records.ndjson` is the better tool for bulk reads (streaming,
+      // resumable, no tar to unpack). The cap stays well above any real
+      // collection purely as a guard against an accidental multi-GB download.
       if (version.recordCount > MAX_EXPORT_RECORDS) {
         return c.json(
           {
             error:
               `This collection has ${version.recordCount.toLocaleString()} records; export is ` +
-              `available below ${MAX_EXPORT_RECORDS.toLocaleString()}. Read it through the ` +
-              `records API instead (GET .../versions/:n/records?after=…), which pages at any ` +
-              `depth, or the manifest endpoint if you only need hashes.`,
+              `available below ${MAX_EXPORT_RECORDS.toLocaleString()}. For bulk reads use ` +
+              `GET .../versions/:n/records.ndjson, which streams the whole version in one ` +
+              `request and resumes with ?after=.`,
             recordCount: version.recordCount,
             maxRecords: MAX_EXPORT_RECORDS,
             statusCode: 413,
@@ -901,10 +902,37 @@ const app = new Hono<AuthEnv>()
         .from(schema.versionRecords)
         .where(eq(schema.versionRecords.versionId, version.id))
 
+      // tar needs each entry's byte length in its header, so an entry cannot be
+      // written from an unbounded stream. Previously every record of a type was
+      // collected into a string[] and joined — which is why export had to be
+      // capped: at 3.1M records the array exhausts the heap, and the join would
+      // exceed V8's maximum string length even if it didn't.
+      //
+      // Instead each type is emitted in bounded parts. A type that fits in one
+      // part keeps the original `records/<Type>.ndjson` name, so every archive
+      // that works today is byte-identical; only types too large for a single
+      // part split into `records/<Type>.0000.ndjson`, `.0001.ndjson`, … Memory is
+      // one part regardless of collection size.
+      const RECORDS_PER_PART = 25_000
+
       for (const { type } of types) {
-        const lines: string[] = []
         let batchCursor: string | null = null
         let batchHasMore = true
+        let partIndex = 0
+        let pending: string[] = []
+
+        const flushPart = (isFinalPart: boolean) => {
+          if (pending.length === 0) return
+          const name =
+            partIndex === 0 && isFinalPart
+              ? `records/${type}.ndjson`
+              : `records/${type}.${String(partIndex).padStart(4, '0')}.ndjson`
+          const buf = Buffer.from(pending.join('\n') + '\n')
+          pack.entry({ name, size: buf.length }, buf)
+          pending = []
+          partIndex++
+        }
+
         while (batchHasMore) {
           // Walk the (version_id, type, record_id) index; record_objects is
           // joined only to pick up the body for the rows on this page.
@@ -934,11 +962,12 @@ const app = new Hono<AuthEnv>()
           const page = batchHasMore ? batch.slice(0, 5000) : batch
           if (page.length > 0) batchCursor = page[page.length - 1]!.recordId
           for (const r of page) {
-            lines.push(JSON.stringify({ id: r.recordId, type: r.type, data: r.data }))
+            pending.push(JSON.stringify({ id: r.recordId, type: r.type, data: r.data }))
           }
+          // Emit whenever a part fills, so `pending` never grows past one part.
+          if (pending.length >= RECORDS_PER_PART) flushPart(false)
         }
-        const buf = Buffer.from(lines.join('\n') + '\n')
-        pack.entry({ name: `records/${type}.ndjson`, size: buf.length }, buf)
+        flushPart(true)
       }
 
       // Add files