Keep saved named-group rosters out of message history

ARCHITECTURE.md

@@ -37,6 +37,7 @@ All paths are relative to `src/blueferry/` unless noted.
 | `grouping.py` | Correlates MAP iMessages with ANCS Messages notification metadata to recover group membership. |
 | `named_groups.py` | Named-group identity keys and saved reply routes. |
 | `confirmed_groups.py` | Persistent confirmed group rosters in the owner-only settings document. |
+| `group_routes.py` | Saved named-group reply rosters in the settings document, outside history retention. |
 | `starred_threads.py` | Persistent starred-conversation keys in the settings document. |
 | `notification_policy.py` | Persistent desktop notification preferences. |
 | `private_preferences.py` | Encrypts a whole preference collection under the storage policy. |
@@ -392,7 +393,9 @@ A change to these rules has to be made in both places.
   threads. Only the backend retains a user-supplied route, and every observed
   sender must remain in it. A sender outside the route raises a roster-change
   warning and disables replies. Routes are local and never modify iPhone
-  groups. Two named groups with the same name share a key because ANCS has no
+  groups. They are preferences in `settings.json`, not history events, so
+  retention and the bounded conversation window never discard a route that
+  is still in use. Two named groups with the same name share a key because ANCS has no
   conversation ID. Versioned keys preserve spelling (NFC, trimmed), and editing
   a roster does not change the key. Legacy name-folded keys remain aliases
   only when history shows a single spelling, and reading history never
@@ -401,8 +404,9 @@ A change to these rules has to be made in both places.
   directories as `0600` SQLite files. Sensitive records, including event kind,
   timestamp, and content, are encrypted with AES-256-GCM under one random key
   held by the Secret Service through libsecret. Clients never handle the key,
-  and keyring lookup attributes are non-sensitive. Starred keys and confirmed
-  rosters in `settings.json` are encrypted per collection.
+  and keyring lookup attributes are non-sensitive. Starred keys, saved group
+  routes, and confirmed rosters in `settings.json` are encrypted per
+  collection.
 - **Fail closed:** passive startup only loads a key from an already unlocked
   collection and never prompts. If the key is missing or wrong, or plaintext
   is unframed, storage becomes unavailable without deleting records while live

README.md

@@ -40,8 +40,12 @@ addresses are retained; editing and syncing contacts updates the grouping.
 
 Group replies are deliberately cautious. Bluetooth does not give BlueFerry a
 reliable group ID or complete roster, so it disables replies when the
-participants are unclear. Named groups may ask you to confirm a local reply
-roster. This does not change the group on the iPhone.
+participants are unclear. For a named group, BlueFerry learns members only from
+the people who send to it, so it asks you to confirm the full reply list once.
+That saved list is kept until you delete the conversation, clear history, or
+change the storage mode; it does not change the group on the iPhone. A message is filed under its group
+only when BlueFerry also receives the iPhone's notification for it; without
+one, it appears in the sender's one-to-one conversation.
 
 ## Install
 
@@ -257,11 +261,12 @@ can also choose unencrypted storage or **Do not retain local data**. Changing
 storage modes clears the existing cache so encrypted and plaintext records are
 never mixed.
 
-Starred conversations and saved group confirmations follow the same storage
-policy as history. Older plaintext preferences are encrypted during upgrade
-when the wallet is available; if it is locked, those old preferences are
-cleared so contact identities are no longer retained in plaintext. You can
-star conversations and confirm group rosters again after unlocking.
+Starred conversations, saved group participants, and group confirmations follow
+the same storage policy as history. Older plaintext preferences are encrypted
+during upgrade when the wallet is available; if it is locked, those old
+preferences are cleared so contact identities are no longer retained in
+plaintext. You can star conversations and confirm group rosters again after
+unlocking.
 
 Configuration lives in `~/.config/blueferry`; local state lives in
 `~/.local/state/blueferry`. Uninstalling packages does not delete either

src/blueferry/backend_operations.py

@@ -29,7 +29,6 @@
     correlate_group_events,
 )
 from blueferry.history import (
-    append_event,
     clear_events,
     delete_event_rows,
     history_revision,
@@ -135,6 +134,16 @@ def discard(self, thread_keys: Sequence[str]) -> None: ...
     def clear(self) -> None: ...
 
 
+class GroupRoutes(Protocol):
+    def routes(self) -> list[dict]: ...
+
+    def save(self, route: dict, *, replacing: Iterable[str] = ()) -> None: ...
+
+    def discard(self, thread_keys: Iterable[str]) -> None: ...
+
+    def clear(self) -> None: ...
+
+
 class ConfirmedGroups(Protocol):
     def matching_rosters(self, rosters: Mapping[str, str]) -> set[str]: ...
 
@@ -162,6 +171,7 @@ class BackendDependencies:
     on_notification_policy_changed: Callable[[], None] | None = None
     starred_threads: StarredThreads | None = None
     confirmed_groups: ConfirmedGroups | None = None
+    group_routes: GroupRoutes | None = None
     storage: StorageSecurity | None = None
     prepare_storage: Callable[[StorageSecurity], Any] | None = None
     on_storage_prepared: Callable[[Any], None] | None = None
@@ -190,13 +200,18 @@ def __init__(
         )
 
     def _build_conversations(self, events: list[dict]) -> list[dict]:
-        return self._project_conversations(events, self.dependencies.contacts, self._starred_keys())
+        return self._project_conversations(
+            events, self.dependencies.contacts, self._starred_keys(), self._group_routes(),
+        )
 
     @staticmethod
     def _project_conversations(
         events: list[dict], contacts: ContactIndex | None, stars: set[str],
+        routes: Sequence[dict] = (),
     ) -> list[dict]:
-        threads = build_threads(events, contacts)
+        # Saved rosters follow history so they override any legacy record
+        # that storage preparation has not yet moved out of the archive.
+        threads = build_threads([*events, *routes], contacts)
         if not stars:
             return threads
         recent = {str(event.get("handle") or "") for event in events[-MAX_CONVERSATION_EVENTS:]}
@@ -209,6 +224,7 @@ def prepare_conversations(
     ) -> None:
         def job_factory() -> Callable[[], list[dict]]:
             stars = self._starred_keys()
+            routes = self._group_routes()
             contacts = self.dependencies.contacts
             resolver = contacts.snapshot() if contacts is not None else None
             storage = self.dependencies.storage
@@ -222,7 +238,7 @@ def project() -> list[dict]:
                     )
                     if reader is not None and reader.status.state == "error":
                         raise CorruptStorageError(reader.status.detail)
-                    return self._project_conversations(events, resolver, stars)
+                    return self._project_conversations(events, resolver, stars, routes)
                 finally:
                     if reader is not None:
                         reader.close()
@@ -426,6 +442,10 @@ def _starred_keys(self) -> set[str]:
             return set()
         return {str(key) for key in store.keys()}
 
+    def _group_routes(self) -> list[dict]:
+        store = self.dependencies.group_routes
+        return store.routes() if store is not None else []
+
     def _group_roster_confirmed(self, thread_key: str, token: str) -> bool:
         if self._confirmed_groups.get(thread_key) == token:
             return True
@@ -598,6 +618,9 @@ def set_group_participants(
         storage = self.dependencies.storage
         if storage is not None and not storage.status.can_write:
             raise NotReadyError(storage.status.detail)
+        routes = self.dependencies.group_routes
+        if routes is None:
+            raise NotReadyError("saved group participants are unavailable")
 
         members: list[str] = []
         for recipient in normalized:
@@ -615,8 +638,8 @@ def set_group_participants(
             "seen_at": datetime.now(timezone.utc).isoformat(),
         }
         try:
-            append_event(route, storage=storage)
-        except (OSError, RuntimeError, ValueError, sqlite3.Error) as error:
+            routes.save(route, replacing=conversation_keys(thread))
+        except (OSError, RuntimeError, ValueError) as error:
             log.error("could not retain named group participants: %s", error)
             raise NotReadyError(
                 "could not retain the group participant list"
@@ -654,6 +677,8 @@ def clear_history(self, confirmed: bool) -> None:
         clear_events()
         if self.dependencies.starred_threads is not None:
             self.dependencies.starred_threads.clear()
+        if self.dependencies.group_routes is not None:
+            self.dependencies.group_routes.clear()
         self._clear_confirmed_groups()
         self.invalidate_conversations()
 
@@ -810,6 +835,8 @@ def delete_threads(
         self._forget_confirmed_groups(resolved_keys | preference_keys)
         if self.dependencies.starred_threads is not None:
             self.dependencies.starred_threads.discard(list(preference_keys))
+        if self.dependencies.group_routes is not None:
+            self.dependencies.group_routes.discard(resolved_keys | preference_keys)
         self.invalidate_conversations()
         return len(selected)
 
@@ -835,6 +862,8 @@ def _prepare_storage_policy(self, value: str) -> str:
             clear_contact_cache()
             if self.dependencies.starred_threads is not None:
                 self.dependencies.starred_threads.clear()
+            if self.dependencies.group_routes is not None:
+                self.dependencies.group_routes.clear()
             self._clear_confirmed_groups()
         return selected
 

src/blueferry/daemon.py

@@ -33,6 +33,7 @@
 from blueferry.contacts import ContactsResolver
 from blueferry.dbus_service import MessagesService, claim_bus_name
 from blueferry.event_dispatcher import EventDispatcher
+from blueferry.group_routes import GroupRoutesStore
 from blueferry.history import (
     history_count,
     mark_event_handles_read,
@@ -107,6 +108,7 @@ def __init__(self) -> None:
         self.notification_policy = NotificationPolicyStore()
         self.starred_threads = StarredThreadsStore(storage=self.storage)
         self.confirmed_groups = ConfirmedGroupsStore(storage=self.storage)
+        self.group_routes = GroupRoutesStore(storage=self.storage)
         self.setup_verification = SetupVerification(config.IPHONE_MAC)
         self.events = EventDispatcher(
             self.contacts,
@@ -302,6 +304,7 @@ def start(self) -> None:
                 on_notification_policy_changed=self._emit_status,
                 starred_threads=self.starred_threads,
                 confirmed_groups=self.confirmed_groups,
+                group_routes=self.group_routes,
                 storage=self.storage,
                 prepare_storage=prepare_storage,
                 on_storage_prepared=self._apply_storage_preparation,

src/blueferry/group_routes.py

@@ -0,0 +1,132 @@
+"""Saved named-group reply rosters in the owner-only settings document.
+
+A roster is user configuration, not message history. Keeping it out of the
+history archive means retention pruning and the bounded conversation window
+can never silently discard it while the group is still in use.
+"""
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from blueferry import config
+from blueferry.limits import MAX_GROUP_ROUTES, MAX_THREAD_KEY_CHARS
+from blueferry.private_preferences import PrivatePreference
+from blueferry.storage_security import StorageSecurity
+
+_SETTINGS_KEY = "named_group_routes"
+_MAX_TEXT_CHARS = 1024
+_MAX_RECIPIENTS = 20
+
+
+def _text(value: object) -> str:
+    text = str(value or "").strip()
+    return text if len(text) <= _MAX_TEXT_CHARS else ""
+
+
+def _strings(value: object) -> list[str] | None:
+    if not isinstance(value, list | tuple) or len(value) > _MAX_RECIPIENTS:
+        return None
+    items = [_text(item) for item in value]
+    return items if all(items) else None
+
+
+def _route(value: object) -> dict | None:
+    """Shape-check one record; NamedGroupRoutes still validates its meaning."""
+    if not isinstance(value, dict):
+        return None
+    key = _text(value.get("group_key"))
+    name = _text(value.get("group_name"))
+    recipients = _strings(value.get("group_recipients"))
+    members = _strings(value.get("group_members") or [])
+    if not key or len(key) > MAX_THREAD_KEY_CHARS or not name or not recipients:
+        return None
+    return {
+        "kind": "group_route",
+        "group_key": key,
+        "group_name": name,
+        "group_members": members or [],
+        "group_recipients": recipients,
+        "seen_at": _text(value.get("seen_at")),
+    }
+
+
+class GroupRoutesStore:
+    """Keep one saved reply roster per named-group key."""
+
+    def __init__(
+        self, path: Path | None = None, *, storage: StorageSecurity | None = None,
+    ) -> None:
+        self._preference = PrivatePreference(
+            path or config.SETTINGS_JSON, _SETTINGS_KEY, storage,
+        )
+
+    def migrate(self) -> None:
+        self._preference.migrate()
+
+    def _mapping(self) -> dict[str, dict]:
+        raw = self._preference.read()
+        if not isinstance(raw, dict):
+            return {}
+        selected: dict[str, dict] = {}
+        for value in raw.values():
+            route = _route(value)
+            if route is None or route["group_key"] in selected:
+                continue
+            selected[route["group_key"]] = route
+            if len(selected) >= MAX_GROUP_ROUTES:
+                break
+        return selected
+
+    def routes(self) -> list[dict]:
+        """Route records oldest first, so the newest save for a name wins."""
+        return sorted(self._mapping().values(), key=lambda route: route["seen_at"])
+
+    def keys(self) -> set[str]:
+        return set(self._mapping())
+
+    def save(self, route: dict, *, replacing: Iterable[str] = ()) -> None:
+        """Store ``route``, dropping records saved under the thread's other keys."""
+        selected = _route(route)
+        if selected is None:
+            raise ValueError("invalid group route")
+        current = self._mapping()
+        for key in (*replacing, selected["group_key"]):
+            current.pop(str(key), None)
+        if len(current) >= MAX_GROUP_ROUTES:
+            raise ValueError(f"at most {MAX_GROUP_ROUTES} group rosters can be saved")
+        current[selected["group_key"]] = selected
+        self._preference.write(current)
+
+    def add_missing(self, routes: Iterable[dict]) -> int:
+        """Adopt legacy records, oldest first, for keys with no saved roster.
+
+        A roster saved in this store is always newer than one kept in history.
+        Among legacy records for one key, the last one wins, as it did when
+        they were read from history.
+        """
+        current = self._mapping()
+        legacy: dict[str, dict] = {}
+        for value in routes:
+            route = _route(value)
+            if route is not None and route["group_key"] not in current:
+                legacy[route["group_key"]] = route
+        added = 0
+        for key, route in legacy.items():
+            if len(current) >= MAX_GROUP_ROUTES:
+                break
+            current[key] = route
+            added += 1
+        if added:
+            self._preference.write(current)
+        return added
+
+    def discard(self, thread_keys: Iterable[str]) -> None:
+        remove = {str(key) for key in thread_keys}
+        current = self._mapping()
+        updated = {key: route for key, route in current.items() if key not in remove}
+        if len(updated) != len(current):
+            self._preference.write(updated)
+
+    def clear(self) -> None:
+        self._preference.clear()

src/blueferry/limits.py

@@ -36,6 +36,7 @@
 MAX_THREAD_DELETE_COUNT = 1_000
 MAX_STARRED_THREADS = 200
 MAX_CONFIRMED_GROUPS = 200
+MAX_GROUP_ROUTES = 200
 MAX_THREAD_KEY_CHARS = 1024
 MAX_GROUP_CONFIRMATION_TOKEN_CHARS = 8192
 MAX_RECENT_QUERY_LIMIT = 200