/-/jump: ?type= filtering, URL fields, hidden tables excluded, query titles

datasette/app.py

@@ -727,7 +727,8 @@ async def refresh_schemas(self, *, force=False):
 
     async def _refresh_schemas(self):
         internal_db = self.get_internal_database()
-        if not self.internal_db_created:
+        first_refresh = not self.internal_db_created
+        if first_refresh:
             await init_internal_db(internal_db)
             await self.apply_metadata_json()
             self.internal_db_created = True
@@ -769,8 +770,12 @@ def delete_stale_database_catalog(conn):
             await internal_db.execute_write_fn(delete_stale_database_catalog)
         for database_name, db in self.databases.items():
             schema_version = (await db.execute("PRAGMA schema_version")).first()[0]
-            # Compare schema versions to see if we should skip it
-            if schema_version == current_schema_versions.get(database_name):
+            # Compare schema versions to see if we should skip it. Always
+            # repopulate on the first refresh, since a persistent internal
+            # database may hold catalog rows calculated using older config.
+            if not first_refresh and schema_version == current_schema_versions.get(
+                database_name
+            ):
                 continue
             await populate_schema_tables(internal_db, db, schema_version)
 

datasette/default_jump_items.py

@@ -54,6 +54,10 @@ async def inner():
                 LEFT JOIN catalog_views
                     ON catalog_views.database_name = allowed_tables.parent
                    AND catalog_views.view_name = allowed_tables.child
+                LEFT JOIN catalog_tables
+                    ON catalog_tables.database_name = allowed_tables.parent
+                   AND catalog_tables.table_name = allowed_tables.child
+                WHERE NOT COALESCE(catalog_tables.hidden, 0)
                 """,
                 params=table_params,
             ),
@@ -72,8 +76,11 @@ async def inner():
                         'query', allowed_queries.child
                     ) AS url,
                     allowed_queries.parent || ' ' || allowed_queries.child AS search_text,
-                    NULL AS display_name
+                    queries.title AS display_name
                 FROM allowed_queries
+                LEFT JOIN queries
+                    ON queries.database_name = allowed_queries.parent
+                   AND queries.name = allowed_queries.child
                 """,
                 params=query_params,
             ),

datasette/utils/internal_db.py

@@ -36,6 +36,7 @@
         table_name TEXT,
         rootpage INTEGER,
         sql TEXT,
+        hidden INTEGER NOT NULL DEFAULT 0,
         PRIMARY KEY (database_name, table_name),
         FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name)
     );
@@ -173,6 +174,12 @@ def initial_internal_schema(db):
     db.executescript(INTERNAL_DB_SCHEMA_SQL)
 
 
+@internal_migrations(name="0002_catalog_tables_hidden")
+def catalog_tables_hidden(db):
+    if "hidden" not in db["catalog_tables"].columns_dict:
+        db["catalog_tables"].add_column("hidden", int, not_null_default=0)
+
+
 async def init_internal_db(db):
     def apply_migrations(conn):
         internal_migrations.apply(SQLiteUtilsDatabase(conn, execute_plugins=False))
@@ -184,6 +191,7 @@ async def populate_schema_tables(internal_db, db, schema_version):
     database_name = db.name
 
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
+    hidden_table_names = set(await db.hidden_table_names())
     views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows
 
     def collect_info(conn):
@@ -202,7 +210,13 @@ def collect_info(conn):
         for table in tables:
             table_name = table["name"]
             tables_to_insert.append(
-                (database_name, table_name, table["rootpage"], table["sql"])
+                (
+                    database_name,
+                    table_name,
+                    table["rootpage"],
+                    table["sql"],
+                    int(table_name in hidden_table_names),
+                )
             )
             columns = table_column_details(conn, table_name)
             columns_to_insert.extend(
@@ -280,8 +294,9 @@ def replace_catalog(conn):
         )
         conn.executemany(
             """
-            INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)
-            values (?, ?, ?, ?)
+            INSERT INTO catalog_tables (
+                database_name, table_name, rootpage, sql, hidden
+            ) values (?, ?, ?, ?, ?)
             """,
             tables_to_insert,
         )

datasette/views/special.py

@@ -1119,8 +1119,9 @@ async def _fragments(self, request):
         return fragments
 
     def _resolve_url(self, url):
+        """Returns (url, kwargs) - kwargs are the datasette.urls arguments, if any"""
         if not url or url.startswith("/"):
-            return url
+            return url, {}
 
         descriptor = json.loads(url)
         if not isinstance(descriptor, dict):
@@ -1140,7 +1141,7 @@ def _resolve_url(self, url):
             raise TypeError(f"datasette.urls.{method_name} is not callable")
         kwargs = {key: value for key, value in descriptor.items() if key != "method"}
         try:
-            return method(**kwargs)
+            return method(**kwargs), kwargs
         except TypeError as ex:
             raise TypeError(
                 f"Invalid arguments for datasette.urls.{method_name}(): {ex}"
@@ -1164,7 +1165,9 @@ def _sort_key(self, row, q):
         }.get(row["type"], 50)
         return (relevance, type_sort, len(display_label), row["label"])
 
-    async def _rows_for_database(self, database_name, indexed_fragments, q, pattern):
+    async def _rows_for_database(
+        self, database_name, indexed_fragments, q, pattern, types=None
+    ):
         params = {"q": q, "pattern": pattern}
         union_parts = []
         for index, fragment in indexed_fragments:
@@ -1186,6 +1189,10 @@ async def _rows_for_database(self, database_name, indexed_fragments, q, pattern)
                 )
             """)
             params.update(fragment_params)
+        types_clause = ""
+        if types:
+            types_clause = "\n        AND type IN (SELECT value FROM json_each(:types))"
+            params["types"] = json.dumps(types)
         sql = f"""
         WITH jump_items AS (
             {" UNION ALL ".join(union_parts)}
@@ -1198,8 +1205,10 @@ async def _rows_for_database(self, database_name, indexed_fragments, q, pattern)
             search_text,
             display_name
         FROM jump_items
-        WHERE :q = ''
-           OR search_text LIKE :pattern COLLATE NOCASE
+        WHERE (
+            :q = ''
+            OR search_text LIKE :pattern COLLATE NOCASE
+        ){types_clause}
         ORDER BY
             CASE
                 WHEN lower(COALESCE(display_name, label)) = lower(:q) THEN 0
@@ -1229,6 +1238,7 @@ async def get(self, request):
         q = request.args.get("q", "").strip()
         terms = q.split()
         pattern = "%" + "%".join(terms) + "%" if terms else "%"
+        types = [t.strip() for t in request.args.getlist("type") if t.strip()]
         fragments = await self._fragments(request)
 
         fragments_by_database = {}
@@ -1241,7 +1251,7 @@ async def get(self, request):
         truncated = False
         for database_name, indexed_fragments in fragments_by_database.items():
             database_rows = await self._rows_for_database(
-                database_name, indexed_fragments, q, pattern
+                database_name, indexed_fragments, q, pattern, types
             )
             if len(database_rows) > 100:
                 truncated = True
@@ -1255,14 +1265,19 @@ async def get(self, request):
 
         matches = []
         for row in rows:
+            url, url_kwargs = self._resolve_url(row["url"])
             match = {
                 "name": row["label"],
-                "url": self._resolve_url(row["url"]),
+                "url": url,
                 "type": row["type"],
                 "description": row["description"],
             }
             if row["display_name"]:
                 match["display_name"] = row["display_name"]
+            # Expose datasette.urls arguments such as database and table as fields
+            for key, value in url_kwargs.items():
+                if key not in match and key != "format":
+                    match[key] = value
             matches.append(match)
 
         return Response.json({"ok": True, "matches": matches, "truncated": truncated})

docs/internals.rst

@@ -2784,7 +2784,7 @@ You can also set the ``DATASETTE_INTERNAL`` environment variable to specify this
     export DATASETTE_INTERNAL=/path/to/internal.db
     datasette mydatabase.db
 
-Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases.
+Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. The ``hidden`` column on ``catalog_tables`` is ``1`` for :ref:`hidden tables <DatabaseView_hidden>`. These tables should not be considered a stable API - they may change between Datasette releases.
 
 Metadata is stored in tables ``metadata_instance``, ``metadata_databases``, ``metadata_resources`` and ``metadata_columns``. Plugins can interact with these tables via the :ref:`get_*_metadata() and set_*_metadata() methods <datasette_get_set_metadata>`.
 
@@ -2832,6 +2832,7 @@ The internal database schema is as follows:
         table_name TEXT,
         rootpage INTEGER,
         sql TEXT,
+        hidden INTEGER NOT NULL DEFAULT 0,
         PRIMARY KEY (database_name, table_name),
         FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name)
     );

docs/introspection.rst

@@ -183,12 +183,16 @@ Shows all actions registered with the permission system, including those added b
 /-/jump
 -------
 
-Returns a JSON list of items that the current actor has permission to view for Datasette's jump menu. By default this includes visible databases, tables, views and stored queries, and plugins can contribute additional items.
+Returns a JSON list of items that the current actor has permission to view for Datasette's jump menu. By default this includes databases, tables, views and stored queries, and plugins can contribute additional items. :ref:`Hidden tables <DatabaseView_hidden>` are not included.
 
-Each item includes a ``type`` string used as a category label in the menu. Items can also include an optional ``description`` with longer text describing that individual result.
+Each item includes a ``type`` string used as a category label in the menu. Items can also include an optional ``description`` with longer text describing that individual result, and an optional ``display_name`` with a human-readable label - for stored queries this is the query ``title``, if one has been set.
+
+Items also include the arguments that were used to construct their URL using the :ref:`internals_datasette_urls` methods. Databases include a ``database`` key, tables and views include ``database`` and ``table`` keys and stored queries include ``database`` and ``query`` keys.
 
 The endpoint supports a ``?q=`` query parameter for filtering items by name.
 
+Use ``?type=`` to return only items of a specific type, for example ``?type=table``. This parameter can be repeated to include multiple types, for example ``?type=table&type=database``. The default item types are ``database``, ``table``, ``view`` and ``query`` - plugins may add their own.
+
 `Jump example <https://latest.datasette.io/-/jump>`_:
 
 .. code-block:: json
@@ -200,19 +204,24 @@ The endpoint supports a ``?q=`` query parameter for filtering items by name.
                 "name": "fixtures",
                 "url": "/fixtures",
                 "type": "database",
-                "description": null
+                "description": null,
+                "database": "fixtures"
             },
             {
                 "name": "fixtures: facetable",
                 "url": "/fixtures/facetable",
                 "type": "table",
-                "description": null
+                "description": null,
+                "database": "fixtures",
+                "table": "facetable"
             },
             {
                 "name": "fixtures: recent_releases",
                 "url": "/fixtures/recent_releases",
                 "type": "query",
-                "description": null
+                "description": null,
+                "database": "fixtures",
+                "query": "recent_releases"
             }
         ],
         "truncated": false
@@ -229,7 +238,9 @@ Search example with ``?q=facet`` returns only items matching ``.*facet.*``:
                 "name": "fixtures: facetable",
                 "url": "/fixtures/facetable",
                 "type": "table",
-                "description": null
+                "description": null,
+                "database": "fixtures",
+                "table": "facetable"
             }
         ],
         "truncated": false

docs/plugin_hooks.rst

@@ -1869,7 +1869,7 @@ The SQL query must return these columns:
     Optional longer text describing this individual item, or ``NULL``. The jump menu displays this below the item's URL when it is present.
 
 ``url``
-    The URL to navigate to when the item is selected. This can be either a string starting with ``/`` or a JSON object describing a call to one of the :ref:`internals_datasette_urls` methods. For example, ``json_object('method', 'table', 'database', 'fixtures', 'table', 'facetable')`` calls ``datasette.urls.table(database='fixtures', table='facetable')``. Unknown methods or invalid named arguments will result in an error.
+    The URL to navigate to when the item is selected. This can be either a string starting with ``/`` or a JSON object describing a call to one of the :ref:`internals_datasette_urls` methods. For example, ``json_object('method', 'table', 'database', 'fixtures', 'table', 'facetable')`` calls ``datasette.urls.table(database='fixtures', table='facetable')``. Unknown methods or invalid named arguments will result in an error. The named arguments (other than ``format``) are also returned as fields on the item in the JSON API, for example ``"database": "fixtures", "table": "facetable"``.
 
 ``search_text``
     Text that should be searched by the ``?q=`` parameter.

tests/test_internal_db.py

@@ -29,7 +29,8 @@ async def test_internal_migrations_recorded(ds_client):
         order by id
         """)
     assert [tuple(row) for row in migrations.rows] == [
-        ("datasette_internal", "0001_initial")
+        ("datasette_internal", "0001_initial"),
+        ("datasette_internal", "0002_catalog_tables_hidden"),
     ]
 
 
@@ -61,7 +62,8 @@ async def test_internal_migrations_adopt_existing_internal_db(tmp_path):
         order by id
         """)
     assert [tuple(row) for row in migrations.rows] == [
-        ("datasette_internal", "0001_initial")
+        ("datasette_internal", "0001_initial"),
+        ("datasette_internal", "0002_catalog_tables_hidden"),
     ]
 
     ds.close()
@@ -73,7 +75,96 @@ async def test_internal_tables(ds_client):
     tables = await internal_db.execute("select * from catalog_tables")
     assert len(tables) > 5
     table = tables.rows[0]
-    assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"}
+    assert set(table.keys()) == {
+        "rootpage",
+        "table_name",
+        "database_name",
+        "sql",
+        "hidden",
+    }
+
+
+@pytest.mark.asyncio
+async def test_internal_tables_hidden(tmp_path):
+    from datasette.app import Datasette
+
+    db_path = str(tmp_path / "data.db")
+    conn = sqlite3.connect(db_path)
+    conn.execute("create table visible (id integer primary key)")
+    conn.execute("create table hidden_by_config (id integer primary key)")
+    conn.commit()
+    conn.close()
+    ds = Datasette(
+        [db_path],
+        config={
+            "databases": {"data": {"tables": {"hidden_by_config": {"hidden": True}}}}
+        },
+    )
+    await ds.invoke_startup()
+    await ds.refresh_schemas(force=True)
+
+    async def hidden_values():
+        rows = await ds.get_internal_database().execute(
+            "select table_name, hidden from catalog_tables "
+            "where database_name = 'data' order by table_name"
+        )
+        return dict(tuple(row) for row in rows.rows)
+
+    assert await hidden_values() == {"hidden_by_config": 1, "visible": 0}
+
+    # Tables created at runtime are updated on the next schema refresh
+    await ds.get_database("data").execute_write(
+        "create virtual table docs_fts using fts5(body)"
+    )
+    await ds.refresh_schemas(force=True)
+    values = await hidden_values()
+    assert values["docs_fts"] == 0
+    assert values["docs_fts_data"] == 1
+    assert values["docs_fts_config"] == 1
+    assert values["visible"] == 0
+    ds.close()
+
+
+@pytest.mark.asyncio
+async def test_internal_tables_hidden_added_to_existing_internal_db(tmp_path):
+    from datasette.app import Datasette
+
+    db_path = str(tmp_path / "data.db")
+    conn = sqlite3.connect(db_path)
+    conn.execute("create table _private (id integer primary key)")
+    conn.commit()
+    conn.close()
+
+    # Internal database created before catalog_tables.hidden existed
+    internal_db_path = str(tmp_path / "internal.db")
+    conn = sqlite3.connect(internal_db_path)
+    conn.executescript(
+        INTERNAL_DB_SCHEMA_SQL.replace("hidden INTEGER NOT NULL DEFAULT 0,", "")
+    )
+    schema_version = (
+        sqlite3.connect(db_path).execute("PRAGMA schema_version").fetchone()[0]
+    )
+    conn.execute(
+        "insert into catalog_databases (database_name, path, is_memory, schema_version) "
+        "values ('data', ?, 0, ?)",
+        [db_path, schema_version],
+    )
+    conn.execute(
+        "insert into catalog_tables (database_name, table_name, rootpage, sql) "
+        "values ('data', '_private', 2, 'create table _private (id integer primary key)')"
+    )
+    conn.commit()
+    conn.close()
+
+    ds = Datasette([db_path], internal=internal_db_path)
+    await ds.invoke_startup()
+    await ds.refresh_schemas(force=True)
+    rows = await ds.get_internal_database().execute(
+        "select table_name, hidden from catalog_tables where database_name = 'data'"
+    )
+    # The catalog is repopulated on startup even though the schema version matches
+    assert [tuple(row) for row in rows.rows] == [("_private", 1)]
+    ds.close()
 
 
 @pytest.mark.asyncio