diff --git a/pgpm/export/__tests__/__snapshots__/export-flow.test.ts.snap b/pgpm/export/__tests__/__snapshots__/export-flow.test.ts.snap index 4b4ee9f5f0..4eb0aff6c7 100644 --- a/pgpm/export/__tests__/__snapshots__/export-flow.test.ts.snap +++ b/pgpm/export/__tests__/__snapshots__/export-flow.test.ts.snap @@ -25,6 +25,7 @@ exports[`Export Flow E2E Export to second workspace should match snapshot for se "migrate/apis.sql", "migrate/app_module.sql", "migrate/catalog_module.sql", + "migrate/catalog_private.apis.sql", "migrate/database.sql", "migrate/database_settings_module.sql", "migrate/domain_module.sql", diff --git a/pgpm/export/src/catalog-projection.ts b/pgpm/export/src/catalog-projection.ts new file mode 100644 index 0000000000..1051c806cb --- /dev/null +++ b/pgpm/export/src/catalog-projection.ts @@ -0,0 +1,52 @@ +import { Parser } from 'csv-to-pg'; + +/** + * Catalog plane projection: catalog_private.apis is trigger-derived from + * routing_public.apis by catalog_private.tg_apis_catalog_sync(). During + * migration replay the sync trigger is skipped (session_replication_role), + * and the catalog tables cannot be queried through the meta API (bare-name + * collisions with routing_public) — so the projection is materialized at + * export time, mirroring the trigger mapping 1:1. resolve_route() needs + * these rows to build resolved_config for api targets. + * + * Both the SQL and GraphQL export flows run this projection so their output + * stays byte-identical (cross-flow parity). + */ +export const projectCatalogApis = async ( + apisRows: Record[] +): Promise => { + if (!apisRows.length) return undefined; + + const projected = apisRows.map((r) => ({ + id: r.id, + owner_scope: 'database', + owner_key: r.database_id, + is_visible: r.is_published ?? false, + database_id: r.database_id, + name: r.name, + dbname: r.dbname, + role_name: r.role_name, + anon_role: r.anon_role, + config: r.config ?? null + })); + + const parser = new Parser({ + schema: 'catalog_private', + table: 'apis', + fields: { + id: 'uuid', + owner_scope: 'text', + owner_key: 'uuid', + is_visible: 'boolean', + database_id: 'uuid', + name: 'text', + dbname: 'text', + role_name: 'text', + anon_role: 'text', + config: 'jsonb' + } + }); + + const parsed = await parser.parse(projected); + return parsed || undefined; +}; diff --git a/pgpm/export/src/export-graphql-meta.ts b/pgpm/export/src/export-graphql-meta.ts index 2bdd3162e9..b8c6d8a134 100644 --- a/pgpm/export/src/export-graphql-meta.ts +++ b/pgpm/export/src/export-graphql-meta.ts @@ -9,6 +9,7 @@ import { Parser } from 'csv-to-pg'; import { toSnakeCase } from 'inflekt'; +import { projectCatalogApis } from './catalog-projection'; import { FieldType, getTimestampDefaultColumnsForTable, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils'; import { GraphQLClient } from './graphql-client'; import { @@ -144,6 +145,23 @@ export const exportGraphQLMeta = async ({ database_id }: ExportGraphQLMetaParams): Promise => { const sql: Record = {}; + // Raw rows per key (post GraphQL→Postgres conversion), kept for derived + // projections (see catalog plane derivation below). + const rawRows: Record[]> = {}; + + // Binding tables (hostname_bindings, route_bindings) carry no database_id; + // tenant ownership flows through domain_id → routing_public.domains. Fetch + // the tenant's domain ids once so those keys can be filtered by + // domainId IN (...). + let domainIds: string[] = []; + if (META_TABLE_ORDER.some((k) => META_TABLE_CONFIG[k]?.filterViaDomainIds)) { + const domainRows = await client.fetchAllNodes<{ id: string }>( + getGraphQLQueryName('domains'), + 'id', + { databaseId: database_id } + ); + domainIds = domainRows.map((r) => r.id); + } const queryAndParse = async (key: string) => { const tableConfig = META_TABLE_CONFIG[key]; @@ -152,8 +170,7 @@ export const exportGraphQLMeta = async ({ // Schema-qualified manifest keys (e.g. catalog_private.apis) // mark tables whose name collides with a table in another plane. GraphQL // type/query names are derived from the bare table name, so these cannot - // be addressed unambiguously through the meta API — only the SQL flow - // exports them. + // be addressed unambiguously through the meta API in a mixed build. if (key.includes('.')) return; // Build fields dynamically: either from hardcoded config or via introspection @@ -167,7 +184,9 @@ export const exportGraphQLMeta = async ({ // The 'database' table is fetched by id, not by database_id const condition = key === 'database' ? { id: database_id } - : { databaseId: database_id }; + : tableConfig.filterViaDomainIds + ? { domainId: domainIds } + : { databaseId: database_id }; try { const rows = await client.fetchAllNodes( @@ -225,6 +244,8 @@ export const exportGraphQLMeta = async ({ if (Object.keys(dynamicFields).length === 0) return; + rawRows[key] = pgRows; + // Omit columnDefaults columns from row data so the Parser never sees them. // configFields already excludes them (via buildDynamicFieldsFromGraphQL), // so dynamicFields won't contain them either — but the pgRow data still does. @@ -279,5 +300,19 @@ export const exportGraphQLMeta = async ({ await Promise.all(keys.map(key => queryAndParse(key))); } + // Catalog plane projection: catalog_private.apis is trigger-derived from + // routing_public.apis (catalog_private.tg_apis_catalog_sync). The sync + // trigger is skipped during migration replay (session_replication_role), + // and the catalog tables can't be queried through the meta API (bare-name + // collisions with routing_public) — so the projection is materialized here. + // Shared with the SQL flow (see catalog-projection.ts) for cross-flow parity. + const apisRows = rawRows['apis']; + if (sql['apis'] && apisRows?.length) { + const parsed = await projectCatalogApis(apisRows); + if (parsed) { + sql['catalog_private.apis'] = parsed; + } + } + return sql; }; diff --git a/pgpm/export/src/export-meta.ts b/pgpm/export/src/export-meta.ts index dc526576f7..25dabd1093 100644 --- a/pgpm/export/src/export-meta.ts +++ b/pgpm/export/src/export-meta.ts @@ -3,6 +3,7 @@ import { Parser } from 'csv-to-pg'; import type { Pool } from 'pg'; import { getPgPool } from 'pg-cache'; +import { projectCatalogApis } from './catalog-projection'; import { FieldType, getTableColumnsWithDefaults, isTimestampDefaultColumn,mapPgTypeToFieldType, META_TABLE_CONFIG, META_TABLE_ORDER, TableConfig } from './export-utils'; /** @@ -84,6 +85,9 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams database: dbname }); const sql: Record = {}; + // Raw rows per key (post column-default stripping), kept for derived + // projections (see catalog plane projection below). + const rawRows: Record[]> = {}; // Cache for dynamically built parsers and their field configs const parsers: Record = {}; @@ -160,6 +164,8 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams } } + rawRows[key] = result.rows; + const parsed = await parser.parse(result.rows); if (parsed) { sql[key] = parsed; @@ -179,8 +185,27 @@ export const exportMeta = async ({ opts, dbname, database_id }: ExportMetaParams // itself, which is keyed by id. for (const key of META_TABLE_ORDER) { const tableConfig = META_TABLE_CONFIG[key]; - const filterColumn = key === 'database' ? 'id' : 'database_id'; - await queryAndParse(key, `SELECT * FROM ${tableConfig.schema}.${tableConfig.table} WHERE ${filterColumn} = $1 ORDER BY id`); + // Binding tables (hostname_bindings, route_bindings) carry no database_id; + // tenant ownership flows through domain_id → routing_public.domains. + const filterSql = key === 'database' + ? 'id = $1' + : tableConfig.filterViaDomainIds + ? 'domain_id IN (SELECT id FROM routing_public.domains WHERE database_id = $1)' + : 'database_id = $1'; + await queryAndParse(key, `SELECT * FROM ${tableConfig.schema}.${tableConfig.table} WHERE ${filterSql} ORDER BY id`); + } + + // Catalog plane projection: catalog_private.apis is trigger-derived from + // routing_public.apis (catalog_private.tg_apis_catalog_sync). The sync + // trigger is skipped during migration replay (session_replication_role), + // so the projection is materialized here. Shared with the GraphQL flow + // (see catalog-projection.ts) for cross-flow parity. + const apisRows = rawRows['apis']; + if (sql['apis'] && apisRows?.length) { + const parsed = await projectCatalogApis(apisRows); + if (parsed) { + sql['catalog_private.apis'] = parsed; + } } return sql; diff --git a/pgpm/export/src/export-utils.ts b/pgpm/export/src/export-utils.ts index 074ee8fe5d..017f297b37 100644 --- a/pgpm/export/src/export-utils.ts +++ b/pgpm/export/src/export-utils.ts @@ -178,6 +178,13 @@ export interface TableConfig { conflictDoNothing?: boolean; typeOverrides?: Record; // only for special types (image, upload, url) that can't be inferred gqlTypeName?: string; // override for GraphQL type name when automatic derivation doesn't match PostGraphile's inflector + /** + * Table has no database_id column; rows belong to a tenant through their + * domain_id FK (e.g. hostname_bindings, route_bindings). The export filters + * them by `domainId in ()` — the tenant's domain ids + * are pre-fetched once per export run. + */ + filterViaDomainIds?: boolean; /** Columns whose values are environment-specific and should be excluded from the * exported INSERT so that the column's DDL DEFAULT applies at deploy time. * Key = column name, Value = the SQL expression the column defaults to (for documentation). @@ -205,6 +212,12 @@ export interface MetaExportTableEntry { * values must come from DDL defaults at deploy time. */ export const META_TABLE_OVERRIDES: Record> = { + hostname_bindings: { + filterViaDomainIds: true + }, + route_bindings: { + filterViaDomainIds: true + }, sites: { typeOverrides: { og_image: 'image', diff --git a/pgpm/export/src/graphql-client.ts b/pgpm/export/src/graphql-client.ts index cf2685c129..ae12bde042 100644 --- a/pgpm/export/src/graphql-client.ts +++ b/pgpm/export/src/graphql-client.ts @@ -176,6 +176,7 @@ export class GraphQLClient { if (condition && Object.keys(condition).length > 0) { const filterParts = Object.entries(condition) .map(([k, v]) => { + if (Array.isArray(v)) return `${k}: { in: [${v.map((item) => `"${item}"`).join(', ')}] }`; if (typeof v === 'string') return `${k}: { equalTo: "${v}" }`; if (typeof v === 'boolean') return `${k}: { equalTo: ${v} }`; if (typeof v === 'number') return `${k}: { equalTo: ${v} }`; diff --git a/pgpm/export/src/meta-export-tables.json b/pgpm/export/src/meta-export-tables.json index 1119a642ff..83efa833ab 100644 --- a/pgpm/export/src/meta-export-tables.json +++ b/pgpm/export/src/meta-export-tables.json @@ -591,6 +591,16 @@ "schema": "routing_public", "table": "routes" }, + { + "key": "hostname_bindings", + "schema": "routing_public", + "table": "hostname_bindings" + }, + { + "key": "route_bindings", + "schema": "routing_public", + "table": "route_bindings" + }, { "key": "site_app_links", "schema": "routing_public", @@ -671,11 +681,6 @@ "schema": "catalog_private", "table": "domains" }, - { - "key": "functions", - "schema": "catalog_private", - "table": "functions" - }, { "key": "namespaces", "schema": "catalog_private", @@ -957,6 +962,12 @@ "created_at", "updated_at" ], + "routing_public.hostname_bindings": [ + "updated_at" + ], + "routing_public.route_bindings": [ + "updated_at" + ], "routing_public.site_app_links": [ "created_at", "updated_at"