Remove Xapian bindings from message ID lookup

.github/workflows/imap.yml

@@ -24,7 +24,7 @@ jobs:
           dovecot-pop3d \
           isync \
           notmuch \
-          afew libnotmuch-dev python3-notmuch2 python3-xapian ssh wget
+          afew libnotmuch-dev python3-notmuch2 ssh wget
 
     - name: Setup Dovecot IMAP server
       run: |

.github/workflows/notmuch-ml.yml

@@ -15,7 +15,7 @@ jobs:
     - uses: actions/checkout@v4
     - run: sudo cp $GITHUB_WORKSPACE/src/notmuch_sync.py /usr/bin/notmuch-sync
 
-    - run: sudo apt-get install -y notmuch libnotmuch-dev python3-notmuch2 python3-xapian wget ssh expect
+    - run: sudo apt-get install -y notmuch libnotmuch-dev python3-notmuch2 wget ssh expect
 
     - run: |
         echo -e "[database]\npath=$GITHUB_WORKSPACE/notmuch-list\nhook_dir=$GITHUB_WORKSPACE/.notmuch\n\n[user]\nname=test\nprimary_email=test@test.com\n\n[new]\ntags=unread;\nignore=\n\n[search]\nexclude_tags=deleted" > ~/.notmuch-config

.github/workflows/python.yml

@@ -23,7 +23,7 @@ jobs:
       uses: actions/setup-python@v3
     - name: Install dependencies
       run: |
-        sudo apt-get install -y notmuch libnotmuch-dev libxapian-dev
+        sudo apt-get install -y notmuch libnotmuch-dev
         python -m pip install --upgrade pip
         pip install flake8 pytest mypy
         pip install -r requirements.txt

README.md

@@ -13,8 +13,7 @@ mbsync-compatible syncing of notmuch databases and mail files.
 
 Assumes that you have [notmuch](https://notmuchmail.org) installed and working.
 Install with e.g. `pip install notmuch-sync`. No configuration is necessary;
-everything is picked up from notmuch. You may however need to install your OS'
-packages for xapian.
+everything is picked up from notmuch.
 
 Before you run `notmuch-sync` for the first time, make sure that notmuch is set
 up correctly (in particular with the correct database path). It is not necessary
@@ -227,10 +226,8 @@ notmuch-sync when this happens.
 Running `notmuch compact` changes the UUID of the database. This means that
 subsequent syncs will abort with an error message.
 
-Like muchsync, notmuch-sync uses xapian directly for some operations for
-efficiency reasons. This means in particular that notmuch-sync assumes that
-value 1 of a xapian document is the message ID and the term "Tghost" is used to
-identify ghost messages.
+Listing all message IDs for deletion synchronization uses notmuch's Python
+bindings and can be expensive for large databases.
 
 notmuch-sync assumes that the remote command does not produce any output except
 for the output produced by the remote notmuch-sync. If you're running a wrapper

pyproject.toml

@@ -16,7 +16,6 @@ authors = [
 requires-python = ">=3.6"
 dependencies = [
     "notmuch2",
-    "xapian-bindings",
 ]
 readme = "README.md"
 license = "BSD-3-Clause"

requirements.txt

@@ -1,4 +1,3 @@
 notmuch2
-xapian-bindings
 pytest
 pytest-shell-utilities

src/notmuch_sync.py

@@ -21,7 +21,6 @@
 from select import select
 
 import notmuch2
-import xapian
 
 logging.basicConfig(format="[{asctime}] {message}", style="{")
 logger = logging.getLogger(__name__)
@@ -521,34 +520,21 @@ def _recv_files():
 
 def get_ids(prefix: str) -> List[str]:
     """
-    Get all message IDs from the notmuch database, using Xapian directly (much
-    faster).
+    Get all message IDs from the notmuch database.
 
     Args:
         prefix (str): Prefix path for filenames (notmuch config database.path).
 
     Returns:
         list: All message IDs.
     """
-    db = xapian.Database(os.path.join(prefix, ".notmuch", "xapian"))
     message_ids = []
 
     logger.info("Getting all message IDs from DB...")
-    ghosts = {p.docid for p in db.postlist("Tghost")} # type: ignore[attr-defined]
-    all_docs = set(range(1, db.get_lastdocid() + 1))
-    for doc_id in all_docs - ghosts:
-        try:
-            doc = db.get_document(doc_id)
-            value = doc.get_value(1)
-            if value:
-                message_ids.append(value.decode("utf-8"))
-        except xapian.DocNotFoundError:
-            pass
-        except RuntimeError:
-            # not entirely sure why, but this seems to happen sometimes
-            continue
-
-    db.close()
+    with notmuch2.Database(path=prefix, mode=notmuch2.Database.MODE.READ_ONLY) as db:
+        for msg in db.messages("*", omit_excluded=notmuch2.Database.EXCLUDE.FALSE):
+            if not msg.ghost and msg.messageid:
+                message_ids.append(msg.messageid)
 
     return message_ids
 
@@ -867,12 +853,13 @@ def sync_local(args: argparse.Namespace) -> None:
             else:
                 remote_changes = (0,0,0,0,0,0)
         finally:
-            ready, _, exc = select([err_remote], [], [], 0)
-            if err_remote is not None and ready and not exc:
-                data = err_remote.read()
-                # getting zero data on EOF
-                if len(data) > 0:
-                    logger.error("Remote error: %s", data)
+            if err_remote is not None:
+                ready, _, exc = select([err_remote], [], [], 0)
+                if ready and not exc:
+                    data = err_remote.read()
+                    # getting zero data on EOF
+                    if len(data) > 0:
+                        logger.error("Remote error: %s", data)
 
             if to_remote is not None:
                 to_remote.close()

test/test.py

@@ -1325,29 +1325,19 @@ def test_sync_deletes_remote_none():
 
 
 def test_get_ids():
-    p1 = lambda: None
-    p1.docid = 1
-    p2 = lambda: None
-    p2.docid = 2
-    p3 = lambda: None
-    p3.docid = 3
-    db = lambda: None
-    db.postlist = MagicMock(return_value=[p1, p2, p3])
-    db.get_lastdocid = MagicMock(return_value=6)
-    db.close = MagicMock()
-    doc = lambda: None
-    doc.get_value = MagicMock()
-    doc.get_value.side_effect = [b"a", b"b", b"c"]
-    db.get_document = MagicMock(return_value=doc)
-
-    with patch("xapian.Database", return_value=db) as xdb:
-        assert ["a", "b", "c"] == ns.get_ids(prefix)
-        xdb.assert_called_once_with(prefix + ".notmuch/xapian")
-        db.postlist.assert_called_once_with("Tghost")
-        db.get_lastdocid.assert_called_once()
-        assert db.get_document.mock_calls == [call(4), call(5), call(6)]
-        assert doc.get_value.mock_calls == [call(1), call(1), call(1)]
-        db.close.assert_called_once()
+    m1 = MagicMock(ghost=False, messageid="a")
+    m2 = MagicMock(ghost=True, messageid="ghost")
+    m3 = MagicMock(ghost=False, messageid="b")
+    db = MagicMock()
+    db.messages.return_value = [m1, m2, m3]
+    mock_ctx = MagicMock()
+    mock_ctx.__enter__.return_value = db
+    mock_ctx.__exit__.return_value = False
+
+    with patch("notmuch2.Database", return_value=mock_ctx) as ndb:
+        assert ["a", "b"] == ns.get_ids(prefix)
+        ndb.assert_called_once_with(path=prefix, mode=notmuch2.Database.MODE.READ_ONLY)
+        db.messages.assert_called_once_with("*", omit_excluded=notmuch2.Database.EXCLUDE.FALSE)
 
 
 def test_sync_mbsync_local_nothing():