fix: search indexing performance and batching

apps/labrinth/.env.docker-compose

@@ -25,9 +25,12 @@ ELASTICSEARCH_INDEX_PREFIX=labrinth
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=elastic
 SEARCH_INDEX_CHUNK_SIZE=5000
+SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5
+SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000
 TYPESENSE_URL=http://localhost:8108
 TYPESENSE_API_KEY=modrinth
 TYPESENSE_INDEX_PREFIX=labrinth
+TYPESENSE_IMPORT_BATCH_SIZE=5000
 
 REDIS_URL=redis://labrinth-redis
 REDIS_MIN_CONNECTIONS=0

apps/labrinth/.env.local

@@ -43,9 +43,12 @@ ELASTICSEARCH_USERNAME=
 ELASTICSEARCH_PASSWORD=
 
 SEARCH_INDEX_CHUNK_SIZE=5000
+SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5
+SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000
 TYPESENSE_URL=http://localhost:8108
 TYPESENSE_API_KEY=modrinth
 TYPESENSE_INDEX_PREFIX=labrinth
+TYPESENSE_IMPORT_BATCH_SIZE=5000
 
 REDIS_URL=redis://localhost
 REDIS_MIN_CONNECTIONS=0

apps/labrinth/src/background_task.rs

@@ -19,7 +19,7 @@ use crate::util::anrok;
 use actix_web::web;
 use clap::ValueEnum;
 use eyre::WrapErr;
-use tracing::info;
+use tracing::{info, instrument};
 
 #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)]
 #[clap(rename_all = "kebab_case")]
@@ -50,6 +50,7 @@ pub enum BackgroundTask {
 
 impl BackgroundTask {
     #[allow(clippy::too_many_arguments)]
+    #[instrument(skip_all, fields(background_task = ?self))]
     pub async fn run(
         self,
         pool: PgPool,

apps/labrinth/src/env.rs

@@ -38,7 +38,7 @@ macro_rules! vars {
                     )]
                     let $field: Option<$ty> = {
                         let mut default = None::<$ty>;
-                        $( default = Some({ $default }.into()); )?
+                        $( default = Some(<$ty>::from({ $default })); )?
 
                         match parse_value::<$ty>(stringify!($field), default) {
                             Ok(value) => Some(value),
@@ -160,9 +160,13 @@ vars! {
     // search
     SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
     SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
+    SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS: u64 = 5u64;
+    SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE: usize = 1000usize;
     TYPESENSE_URL: String = "http://localhost:8108";
     TYPESENSE_API_KEY: String = "modrinth";
     TYPESENSE_INDEX_PREFIX: String = "labrinth";
+    TYPESENSE_IMPORT_BATCH_SIZE: usize = 5000usize;
+    TYPESENSE_DELETE_BATCH_SIZE: usize = 10_000usize;
 
     // storage
     STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local;

apps/labrinth/src/lib.rs

@@ -131,30 +131,6 @@ pub fn app_setup(
     ));
 
     if enable_background_tasks {
-        // The interval in seconds at which the local database is indexed
-        // for searching.  Defaults to 1 hour if unset.
-        let local_index_interval =
-            Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL);
-        let pool_ref = pool.clone();
-        let redis_pool_ref = redis_pool.clone();
-        let search_backend_ref = search_backend.clone();
-        scheduler.run(local_index_interval, move || {
-            let pool_ref = pool_ref.clone();
-            let redis_pool_ref = redis_pool_ref.clone();
-            let search_backend = search_backend_ref.clone();
-            async move {
-                if let Err(err) = background_task::index_search(
-                    pool_ref,
-                    redis_pool_ref,
-                    search_backend,
-                )
-                .await
-                {
-                    warn!("Failed to index search: {err:?}");
-                }
-            }
-        });
-
         // Changes statuses of scheduled projects/versions
         let pool_ref = pool.clone();
         // TODO: Clear cache when these are run

apps/labrinth/src/routes/v3/projects.rs

@@ -29,7 +29,8 @@ use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
 use crate::routes::internal::delphi;
 use crate::search::{
-    SearchBackend, SearchQuery, SearchRequest, SearchResults, SearchState,
+    SearchBackend, SearchIndexUpdate, SearchQuery, SearchRequest,
+    SearchResults, SearchState,
 };
 use crate::util::error::Context;
 use crate::util::img;
@@ -77,6 +78,23 @@ pub async fn clear_project_cache_and_queue_search(
     project_id: db_ids::DBProjectId,
     slug: Option<String>,
     clear_dependencies: Option<bool>,
+) -> Result<(), ApiError> {
+    clear_project_cache_and_queue_search_inner(
+        redis,
+        search_state,
+        project_id,
+        slug,
+        clear_dependencies,
+    )
+    .await
+}
+
+pub async fn clear_project_cache_and_queue_search_inner(
+    redis: &RedisPool,
+    search_state: &SearchState,
+    project_id: db_ids::DBProjectId,
+    slug: Option<String>,
+    clear_dependencies: Option<bool>,
 ) -> Result<(), ApiError> {
     db_models::DBProject::clear_cache(
         project_id,
@@ -85,11 +103,36 @@ pub async fn clear_project_cache_and_queue_search(
         redis,
     )
     .await?;
+
     search_state.queue.push(project_id.into()).await;
 
     Ok(())
 }
 
+pub async fn clear_project_cache_and_queue_search_versions(
+    redis: &RedisPool,
+    search_state: &SearchState,
+    project_id: db_ids::DBProjectId,
+    slug: Option<String>,
+    clear_dependencies: Option<bool>,
+    version_ids: impl IntoIterator<Item = VersionId>,
+) -> Result<(), ApiError> {
+    db_models::DBProject::clear_cache(
+        project_id,
+        slug,
+        clear_dependencies,
+        redis,
+    )
+    .await?;
+
+    search_state
+        .queue
+        .push_versions(project_id.into(), version_ids)
+        .await;
+
+    Ok(())
+}
+
 #[derive(Deserialize, Validate)]
 pub struct RandomProjects {
     #[validate(range(min = 1, max = 100))]
@@ -1098,6 +1141,9 @@ pub async fn project_edit_internal(
         Ok(())
     }
 
+    let reindex_version_project_types =
+        new_project.minecraft_java_server.is_some();
+
     update(
         &mut transaction,
         id,
@@ -1167,31 +1213,36 @@ pub async fn project_edit_internal(
 
     transaction.commit().await?;
 
-    clear_project_cache_and_queue_search(
-        &redis,
-        &search_state,
-        project_item.inner.id,
-        project_item.inner.slug,
-        None,
-    )
-    .await?;
+    if reindex_version_project_types {
+        clear_project_cache_and_queue_search_versions(
+            &redis,
+            &search_state,
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+            project_item.versions.iter().copied().map(VersionId::from),
+        )
+        .await?;
+    } else {
+        clear_project_cache_and_queue_search(
+            &redis,
+            &search_state,
+            project_item.inner.id,
+            project_item.inner.slug,
+            None,
+        )
+        .await?;
+    }
 
     // Remove no longer searchable projects from search index
     if let (true, Some(false)) = (
         project_item.inner.status.is_searchable(),
         new_project.status.map(|status| status.is_searchable()),
     ) {
         search_state
-            .backend
-            .remove_documents(
-                &project_item
-                    .versions
-                    .into_iter()
-                    .map(|x| x.into())
-                    .collect::<Vec<_>>(),
-            )
-            .await
-            .wrap_internal_err("failed to remove documents")?;
+            .queue
+            .push_project_removal(project_item.inner.id.into())
+            .await;
     }
 
     Ok(HttpResponse::NoContent().body(""))
@@ -2791,27 +2842,27 @@ pub async fn project_delete_internal(
         .await
         .wrap_internal_err("failed to commit transaction")?;
 
-    search_state
-        .backend
-        .remove_documents(
-            &project
-                .versions
-                .into_iter()
-                .map(|x| x.into())
-                .collect::<Vec<_>>(),
-        )
-        .await
-        .wrap_internal_err("failed to remove project version documents")?;
-
     if result.is_some() {
-        clear_project_cache_and_queue_search(
-            &redis,
-            &search_state,
+        db_models::DBProject::clear_cache(
             project.inner.id,
             project.inner.slug,
             None,
+            &redis,
         )
         .await?;
+        let project_id = project.inner.id.into();
+        search_state
+            .backend
+            .apply_update(SearchIndexUpdate {
+                removed_projects: std::slice::from_ref(&project_id),
+                ..SearchIndexUpdate::default()
+            })
+            .await
+            .wrap_internal_err("failed to remove project from search index")?;
+        search_state
+            .queue
+            .push_project_removal(project.inner.id.into())
+            .await;
         Ok(())
     } else {
         Err(ApiError::NotFound)