diff --git a/cmd/ledgerctl/store/bootstrap_retry_test.go b/cmd/ledgerctl/store/bootstrap_retry_test.go index 1ba971a681..f8e18e9671 100644 --- a/cmd/ledgerctl/store/bootstrap_retry_test.go +++ b/cmd/ledgerctl/store/bootstrap_retry_test.go @@ -34,6 +34,7 @@ import ( "github.com/formancehq/ledger/v3/internal/infra/backup" "github.com/formancehq/ledger/v3/internal/infra/cache" "github.com/formancehq/ledger/v3/internal/infra/health" + "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/node" "github.com/formancehq/ledger/v3/internal/infra/plan" "github.com/formancehq/ledger/v3/internal/infra/state" @@ -61,7 +62,7 @@ func newBootstrapRetryAdmission(t *testing.T, store *dal.Store) *admission.Admis notifier.EXPECT().NotifyConfigChanged().AnyTimes() machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keys, shared, notifier, nil, - "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, state.NewRecovery(machine, store).RecoverState()) next := machine.State.LastAppliedIndex + 1 @@ -90,7 +91,7 @@ func newBootstrapRetryAdmission(t *testing.T, store *dal.Store) *admission.Admis return future, nil }).AnyTimes() - return admission.NewAdmission(store, logger, proposer, + return admission.NewAdmission(store, logger, proposer, &membership.Coordinator{}, builder, meters, gate, keys, shared, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) } diff --git a/cmd/server/server.go b/cmd/server/server.go index da1916685a..f8c2237ebf 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -873,6 +873,7 @@ func discoverPeersFromCluster(raftAddr string, tlsCfg bootstrap.TLSConfig, clust Address: p.GetRaftAddress(), ServiceAddress: p.GetServiceAddress(), InstanceID: p.GetInstanceId(), + Capabilities: append([]uint32(nil), p.GetCapabilities()...), }) } diff --git a/docs/ops/cluster-operations.md b/docs/ops/cluster-operations.md index 108c41f00a..f83bf84062 100644 --- a/docs/ops/cluster-operations.md +++ b/docs/ops/cluster-operations.md @@ -722,6 +722,6 @@ After the reset, the preloader falls back to Pebble reads (0xF1 zone) for all ca - [Data Flows](../technical/architecture/data-flows.md) - Spool mechanics, entry processing - [Spool](../technical/architecture/subsystems/storage/spool.md) - Spool technical implementation - [Deployment](./deployment.md) - Helm chart configuration, CLI flags -- [CLI Reference](./cli.md) - cluster commands +- [CLI Reference](./cli.md) - `cluster promote-learner` and other cluster commands - [Memory Management](./memory.md) - Cache rotation threshold memory impact - [Availability and Disaster Recovery](./availability-and-recovery.md) - Failure-domain placement and quorum-loss recovery diff --git a/docs/ops/upgrades.md b/docs/ops/upgrades.md index 24168a4226..370cf3637d 100644 --- a/docs/ops/upgrades.md +++ b/docs/ops/upgrades.md @@ -70,6 +70,20 @@ through each adjacent RC in order; do not jump to the newest RC. If the source cannot be identified or the current Operator already rejects the pinning step, stop and plan a separately validated migration. Do not delete the StatefulSet or PVCs to bypass the source check. +### Maintainer-declared CLI break + +The `ledgerctl cluster add-learner` command is intentionally removed from the +capability-evidence release. Existing scripts or runbooks invoking that command +must be updated; the command is not kept as a compatibility alias and no +protocol-version bump is required because the service gRPC contract is +unchanged. + +To add a new node, configure its peer seed and start the Ledger binary. The +bootstrap sequence calls `JoinAsLearner` and advertises the capabilities +compiled into that binary. A node restarting with an existing WAL uses +`RefreshMembership` to publish updated endpoints and capability evidence. +The administrative `AddLearner` RPC remains an internal recovery API, but it is +not a supported replacement for the removed CLI workflow. Before tagging a new release, verify the upgrade with state created by the supported source releases. Cover affected Pebble records, Raft log and snapshot @@ -328,6 +342,16 @@ validation passes. For RC.1 to RC.2, follow the [maintenance-barrier rolling procedure](#account-metadata-address-validation). For RC.3 to RC.4, use the [operator-required stopped restart](#operator-required-stopped-restart). +Capability metadata is the prerequisite for any future incompatible Raft entry, +execution-plan, replay, or snapshot format. This release does not support +mixed-version activation: stop every Ledger node, deploy the barrier-aware +binary everywhere, restart the complete cluster, and only then enable the new +feature. The capability reconciler does not arm features from membership +advertisements. See the [capability contract](../technical/architecture/subsystems/consensus/capabilities.md) +for the persisted replay boundary and coordinated rollout. + +Use this for RC updates after the mixed-version validation above passes: + 1. Record the current leader and voter set with `ledgerctl cluster status`. 2. Restart one follower on the target binary. 3. Wait until it reports normal, has caught up to the leader's commit index, @@ -527,25 +551,27 @@ installation, backup/restore, rollback, or the dormant stopped-upgrade path. The pinned binary-level rc.1 test and relevant model, restore and migration suites remain separate gates. -The same harness has a separate, future `rc3-to-rc4` transition: +The same harness has a separate `rc3-to-rc4` transition for the mandatory +operator-required stopped restart: ```bash UPGRADE_TRANSITION=rc3-to-rc4 RELEASE_BRANCH=release/v3.0 just test-release-operator-upgrade ``` -Run it against a release branch whose operator lists `v3.0.0-rc.4` as a -mandatory stop boundary. The target operator and Ledger are still built from -one exact release-branch HEAD and tagged locally as rc.4; the harness publishes no tag +Run it after `v3.0.0-rc.3` has been published, against a release branch whose +operator lists `v3.0.0-rc.4` as a mandatory stop boundary. The target operator +and Ledger are built from one exact release-branch HEAD and tagged locally as +rc.4; the harness publishes no tag or image. This path requires the operator's drain barrier to advance past `Draining`, observes the StatefulSet and all old Pods absent before accepting any target Pod, and checks that the replacement StatefulSet has a new UID while PVC UIDs persist. It restarts a target voter to exercise replay, verifies the pre-upgrade account, commits another transaction, checks Raft progress and runs each voter's audit/derived-state checker after drain release. It fails -against an operator without the rc.4 boundary, so it is a release gate for the -format-change PR, not evidence that rc.3 itself needs a stop. The rc.4 release -still needs format-specific migration, snapshot, backup/restore and rollback -validation outside this scenario. +against an operator without the rc.4 boundary or if the mandatory stop is +skipped, so it is a release gate for the format-change PR, not evidence that +rc.3 itself needs a stop. The rc.4 release still needs format-specific migration, +snapshot, backup/restore and rollback validation outside this scenario. ## Coordinated Restart diff --git a/docs/technical/architecture/subsystems/consensus/README.md b/docs/technical/architecture/subsystems/consensus/README.md index 30670985ca..c3de13b9f5 100644 --- a/docs/technical/architecture/subsystems/consensus/README.md +++ b/docs/technical/architecture/subsystems/consensus/README.md @@ -13,6 +13,7 @@ The replication and ordering layer (`internal/infra/node`, `internal/infra/trans | [hybrid-logical-clock.md](hybrid-logical-clock.md) | Monotonic HLC timestamps across leader changes and clock skew. | | [removed-member-registry.md](removed-member-registry.md) | Universal member-identity invariant, committed-only peer routing, and replicated `(nodeID, instanceID)` set that prevents a removed member from silently rejoining and being auto-promoted; replay-safe per-event admission barriers, lookup costs, removal write ordering and durable restart states. | | [controlled-shutdown.md](controlled-shutdown.md) | Raft-ordered admission cutoff, local persistence confirmation, restart behavior, rolling-upgrade capability gate, and operator completion contract. | +| [capabilities.md](capabilities.md) | Stable capability declarations, persisted activation policy, membership evidence, and mixed-version rollout boundaries. | ## Related diff --git a/docs/technical/architecture/subsystems/consensus/capabilities.md b/docs/technical/architecture/subsystems/consensus/capabilities.md new file mode 100644 index 0000000000..b84cef4408 --- /dev/null +++ b/docs/technical/architecture/subsystems/consensus/capabilities.md @@ -0,0 +1,179 @@ +# Coordinated-upgrade capabilities + +Ledger's capability metadata is the protocol foundation for future incompatible +features deployed with a coordinated stop-and-restart. It is separate from the +client-facing gRPC protocol revision. This release does not implement a +mixed-version rolling activation path. + +## Wire contract + +`common.ClusterPolicy` carries the replicated activation state. +`raft.Proposal.required_capabilities` carries stable producer-owned capability +IDs. The fields are optional and therefore backward-readable by binaries that +predate this foundation. IDs are never reused; semantic dependencies must be +declared even when they do not add bytes to a proposal. + +The producer must rebuild its plan first, derive the capabilities from that +final plan, stamp `required_capabilities`, and pass the typed result alongside +the serialized proposal. Admission reuses that result without decoding the +proposal a second time. A payload that depends on a capability but omits its +declaration is invalid; the shared builder and stamping helper are the +enforcement boundary for producers. + +## Admission and membership + +The pure decision contract in `internal/infra/capabilities` evaluates every +current voter and learner. A member is represented by `(node_id, instance_id)`; +an identity refresh is a new incarnation and cannot silently inherit evidence +from the old one. Capability activation uses committed support evidence, not a +live health poll: a member that has already acknowledged the capability may be +temporarily unreachable when the activation is committed. Reachability still +blocks admission of capability-dependent writes until that member is reachable +again. Once a capability is active, every new membership announcement must +include that capability. + +A proposal that declares a capability is admitted only when that capability is +active and every member advertises it. Proposals with no declared capability +remain unaffected. FSM apply never consults node-local capability flags. +Membership evidence is an authenticated cluster fact, not a proof of code +inspection: a node can only cause activation for IDs in the leader's local +registry, and malformed (zero, duplicate, or unsorted) declarations are +rejected. Operators must therefore treat a compromised binary as outside the +trust model of this barrier. + +## Capabilities that change FSM semantics + +A capability can describe a protocol or semantic behavior, not only a payload +feature. For example, `MirrorIngestRejectInvalidAssetV2` can identify the +behavior that rejects an asset which an older FSM accepted. + +The capability has two roles in this case: + +1. The replicated `ClusterPolicy` controls whether the cluster may accept the + behavior. +2. The order declares the capability that selects the behavior during FSM + apply. + +The serialized proposal currently carries `required_capabilities` at proposal +scope. That is sufficient only when every order in the proposal has the same +semantic contract. A proposal can batch multiple orders, so a future change +that mixes old and new semantics must either add a persisted marker to each +order (and dispatch apply from that marker) or reject mixed-semantic batches. +The conceptual marker for a homogeneous proposal is: + +```text +old proposal: required_capabilities = [] +new proposal: required_capabilities = [MirrorIngestRejectInvalidAssetV2] +``` + +The FSM must use the persisted declaration to select the behavior. It must not +infer the behavior from the current policy alone. An old order can be replayed +after the capability becomes active, and it must keep its old semantics. + +If a producer cannot mark the new orders, the change must remain backward +compatible or add an explicit semantic version to the order. A capability by +itself is an admission barrier; it is not a historical replay version. + +## Adding a protocol-breaking capability + +Use this workflow when a change can make an order apply differently on two +binary versions. + +1. Name the semantic contract. Describe the old behavior, the new behavior, + and the orders that use each behavior. +2. Allocate a new capability ID. Never reuse an ID for a different contract. +3. Add the capability to the binary registry and derive it from the final + proposal in `RequiredCapabilities`. +4. Stamp the capability on every new order that needs the new behavior. Do not + infer it from the current policy during FSM apply. +5. Keep the old behavior for orders that do not carry the capability. Add + replay tests that apply both order forms with the new binary. +6. Add admission tests for an inactive capability, an active capability, and a + member that does not advertise it. +7. Deploy the compatible binary to all current members. Let each restarted + member refresh its membership evidence before activating the capability. +8. Activate the capability through the replicated `ClusterPolicy`. The FSM + rejects a new member or membership refresh that omits an active capability. +9. Keep the capability active after activation. The policy processor rejects + removal of an active capability. +10. Update this document in the same change. Record the capability ID, the + semantic contract, the replay rule, and the rollout order. + +Do not add a capability for a bug fix that restores the intended behavior when +the old behavior was invalid. Add one when valid historical orders need to keep +their old behavior while new orders use a different rule. + +## Rollout and recovery + +Deploy the barrier-aware binary to every member, stop the cluster, and restart +the complete cluster before enabling an incompatible feature. An old leader +cannot be made safe by upgrading only the other nodes. Restart, snapshot +install, restore, and downgrade code must preserve the policy. Membership +changes are not used as a rolling activation mechanism. `Node.Propose` evaluates the authoritative +policy while holding the proposal and index-tracker guards, before reserving a +predicted index or queueing work; this prevents a rejected proposal from +leaving a hole for queued successors. Removing a member remains allowed; +adding, promoting, or refreshing one without all active capabilities is +rejected. + +`Node.Propose` is the single final policy boundary for every producer, +including mirror, events, maintenance, and backup flows; the shared plan +runner does not duplicate this check. Callers must still stamp their proposal +before use. Membership `ConfChange` operations carry and validate their +capability evidence separately. + +## Admission cost budget and evidence + +The proposal gate has a deliberately bounded cost shape. For a proposal that +declares `r` capabilities and a current membership of `m` voters and learners, +the pure decision is `O(r + m*r)` and allocates only its normalized input and +blocker report. The membership snapshot is copied once per gate evaluation; +the gate must not perform network calls, wait for Raft, or hold a storage +transaction while evaluating the result. Policy decoding and the snapshot +copy remain serialized by the proposal admission boundary, so they are part +of the write-path budget rather than background work. + +The deterministic behavior matrix is covered by +`internal/infra/capabilities/capabilities_test.go`: empty declarations, +inactive policy, active policy with converged voters and learners, missing +evidence, and unreachable members. The representative cost shape is measured +by: + +```text +go test ./internal/infra/capabilities -run '^$' -bench BenchmarkEvaluate -benchmem -count=5 +``` + +The benchmark uses three and seven members with zero, one, and four required +capabilities. Any future capability registry or policy-provider change must +rerun it and include the result in the change review. A regression must be +explained against the `m*r` bound; if the bound no longer fits the admission +budget, move decoding/snapshot preparation outside the serialized section and +pass an immutable admission snapshot instead of widening the lock. + +## Restart refresh + +A normal restart does not run `JoinAsLearner`: the node restores its existing +WAL `ConfState` and keeps the same `(node_id, instance_id)` incarnation. After +the node and its Raft gRPC server are started, a node whose WAL carries +`CLUSTER_JOINED` sends an internal `RefreshMembership` request to a configured +peer. Followers forward that request to the current leader over the Raft gRPC +server. The node can therefore receive inbound Raft traffic before the refresh +commits; the refresh is an evidence update, not a pre-traffic barrier. During +the rc.2/rc.3 rolling upgrade, an older peer that returns `Unimplemented` is +treated as transient and the node keeps retrying in the background until a +compatible peer is available. The capability-dependent protocol cutover is +reserved for the coordinated rc.3/rc.4 restart, after every node runs the +barrier-aware binary. +The leader validates the identity and capability evidence against the active +policy, then proposes a +`ConfChangeUpdateNode` containing the current endpoints and capabilities. + +Only the committed change is authoritative. The update is replicated through +Raft and applied to every member's peer store; a local restart-time upsert is +not sufficient to activate a capability for admission. A fresh WAL instead +uses `JoinAsLearner` and `ConfChangeAddLearnerNode`. + +The cluster-policy reconciler preserves committed capability activations while +reconciling ordinary policy configuration; it does not arm capabilities from +membership advertisements. A future feature must define its activation policy +explicitly and enable it only after the coordinated restart has completed. diff --git a/docs/technical/architecture/subsystems/fsm/preload.md b/docs/technical/architecture/subsystems/fsm/preload.md index fd47547c3c..428a87a464 100644 --- a/docs/technical/architecture/subsystems/fsm/preload.md +++ b/docs/technical/architecture/subsystems/fsm/preload.md @@ -203,7 +203,7 @@ The bitset lives on `OrderTechnical` (`misc/proto/raft_cmd.proto:64`) and `Techn Between the moment a producer resolves `(key=K, value=V)` and the moment the FSM applies the proposal at Raft index `I`, another proposal may mutate `K`. The preloaded `V` would then be stale at apply. -`AppendProposalPredictedIndex(data []byte, index uint64) []byte` (`internal/infra/plan/predicted_index.go`) stamps the expected index onto the already-marshalled proposal by appending the raw wire encoding of `Proposal.predicted_index` (a `fixed64`, under the field number read from the descriptor at init rather than a hardcoded literal); it is a no-op when the index is 0. Appending raw bytes avoids re-marshalling a possibly-megabyte `Proposal` while holding the proposal lock. That is the common path only: when the guard finds the boundary shifted it must re-marshal the rebuilt proposal anyway, and the fresh marshal serializes `PredictedIndex` inline, so the append is skipped (`internal/infra/plan/runner.go` — the index is assigned to `cmd` before the branch, then either re-marshalled or appended, never both). +`Proposal.SetPredictedIndex(index)` (`internal/infra/node/proposal.go`) stamps the expected index onto the already-marshalled proposal by appending the raw wire encoding of `Proposal.predicted_index` (a `fixed64`, under the field number read from the descriptor at init rather than a hardcoded literal); it is a no-op when the index is 0. Appending raw bytes avoids re-marshalling a possibly-megabyte `Proposal` while holding the proposal lock. The full proposal is first marshalled with `PredictedIndex = 0` outside the lock on both the common and no-preload paths. After the guard is acquired, `SetPredictedIndex` appends the bounded field. If the guard detects a boundary shift, the rebuilt command is re-marshalled under the guard and the same `SetPredictedIndex` patch is then applied to the rebuilt bytes. At apply, the FSM compares the predicted index against the actual one. A mismatch means the observed cache state may no longer hold, so the proposal is rejected with `ErrStaleProposal` and the producer rebuilds and retries. @@ -252,12 +252,12 @@ sequenceDiagram P->>P: marshal — outside the critical section P->>B: AcquireProposalGuard: lock IndexTracker, re-check Gen(nextIndex) B-->>P: ProposalGuard - P->>P: cmd.PredictedIndex = TrackerNext() (index known under the lock) + P->>P: Proposal.SetPredictedIndex(TrackerNext()) (bounded patch under the lock) alt boundary shifted between Build and guard (rare) - B-->>P: rebuilt ExecutionPlan → re-apply bits, re-marshal under the guard - Note over P: the re-marshal serializes PredictedIndex inline — no append + B-->>P: rebuilt ExecutionPlan → re-apply bits, re-marshal with index zero + P->>P: Proposal.SetPredictedIndex(TrackerNext()) else common path - P->>P: AppendProposalPredictedIndex onto the pre-marshalled buffer + P->>P: Proposal.SetPredictedIndex(TrackerNext()) onto the pre-marshalled buffer end P->>P: Propose under the guard Note over P,FSM: Raft commit (no Pebble reads on the hot path) diff --git a/docs/technical/audits/raft-membership-leadership.json b/docs/technical/audits/raft-membership-leadership.json index 258971cdf6..fe50e3ae78 100644 --- a/docs/technical/audits/raft-membership-leadership.json +++ b/docs/technical/audits/raft-membership-leadership.json @@ -93,7 +93,8 @@ "Every MsgSnap the leader hands to the transport must be reported to its RawNode with ReportSnapshot for the destination peer. etcd/raft pauses all appends to a peer in StateSnapshot, so a lost MsgSnap or an acknowledgement that arrives after the leader compacted past the sent snapshot must not strand the peer without entries for the rest of the leader's term.", "Controlled drain preflights every current member for protocol support, fences prior membership changes before marker enqueue, and rejects marker or release proposals if leadership term changed since preflight. It blocks normal membership changes from enqueue through applied release. Its completion certificate requires matching current membership and each captured member locally persisting the same cutoff; leader Match alone is insufficient.", "After a full-stop upgrade, production Raft writers serialize ExecutionBatch and SetClusterPolicyProposal payloads directly. No source binary may coexist with the target or reopen its WAL after new writes. Historical replay must preserve exact audit bytes and hashes. Native policy controls are unaudited operational state and must not be lowered to orders. Compare final state across encodings, verify each audit chain and exact policy omission, and preserve the historical prefix byte-for-byte. Incremental manifests must preserve policy-only changes as well as mixed legacy/native deltas. Both forms preserve coverage targets, checkpoint boundaries, technical-only HLC behavior, and post-checkpoint policy restore. Preserve legacy decoding and test reopening the released store/WAL, replaying retained legacy entries, writing new payloads, and restarting.", - "Maintenance must fsync a snapshot file, persist its matching WAL snapshot record, and only then publish the snapshot in memory or compact. A WAL-record failure leaves the file orphaned, keeps the previous snapshot authoritative, and allows an idle retry; compaction remains behind the minimum of the durable FSM cursor and the Raft apply-ack cursor." + "Maintenance must fsync a snapshot file, persist its matching WAL snapshot record, and only then publish the snapshot in memory or compact. A WAL-record failure leaves the file orphaned, keeps the previous snapshot authoritative, and allows an idle retry; compaction remains behind the minimum of the durable FSM cursor and the Raft apply-ack cursor.", + "Capability evidence and policy transitions are replicated membership state: refresh, bootstrap join, promotion, and policy activation must preserve capability evidence, identity, and rolling-upgrade compatibility. An old peer returning Unimplemented for RefreshMembership must not prevent startup, while an active policy must not admit a member or proposal without committed evidence." ], "adversarial_questions": [ "When production learner join preflight commits membership before the joining Raft listener starts, does a failed first optional-TLS probe retain both the committed endpoint and a live retry owner? After listener startup, can ordinary traffic recover replication, catch-up and automatic promotion without re-registration or voter restart, while removal and shutdown terminate pending retries?", @@ -116,7 +117,8 @@ "Can snapshot installation discard entries carried by the same Ready, and what in-memory snapshot/cache state and durable WAL prefix remain if candidate WAL preparation or either directory rename fails before storage acknowledgement?", "After the leader sends a MsgSnap, can the peer's Progress remain in StateSnapshot indefinitely because the snapshot was lost, the peer restarted, or leader maintenance compacted past the sent index before the acknowledgement, leaving a node that reports a normal sync status but never advances?", "Can a mixed-version member pass preflight, or can a membership change slip between preflight and marker application? If leadership is lost and regained after preflight while another leader changes membership, does the old-term marker or release get rejected before Raft append? Can a timed-out marker, lost member, leader transfer or force removal make the operator accept an obsolete captured set or a follower that has not persisted the cutoff?", - "If snapshot-file persistence succeeds but the matching WAL snapshot-record append fails, does the in-memory snapshot remain unchanged, does an idle retry recreate the same snapshot, and can any maintenance tick compact before the record is durable?" + "If snapshot-file persistence succeeds but the matching WAL snapshot-record append fails, does the in-memory snapshot remain unchanged, does an idle retry recreate the same snapshot, and can any maintenance tick compact before the record is durable?", + "During a follower-first rolling upgrade, what happens when RefreshMembership reaches an older peer that returns Unimplemented? Does startup remain available with conservative evidence and eventual retry, and does a later committed refresh converge the membership row? Can capability activation or declaration be applied against stale or pre-refresh evidence, or bypass the authoritative policy gate?" ], "dynamic_checks_to_consider": [ "hold a fresh learner's listener closed through production join preflight with optional TLS, record the current term, committed addition index, durable membership and valid instance identity, then open the listener and require autonomous replication and promotion without another registration; independently test initial probe fail-then-success, actual Raft stream delivery, endpoint replacement, and pending-owner removal/shutdown (internal/bootstrap/join_transport_recovery_test.go and internal/infra/node/transport_initial_probe_test.go)", @@ -141,6 +143,7 @@ "drive a real leader RawNode through sendReadyMessages and finishReady while it sends a MsgSnap, compact its log past that snapshot, then deliver the late acknowledgement and require a fresh MsgSnap instead of a paused peer (internal/infra/node/leader_snapshot_report_test.go)", "Exercise old-member Unimplemented preflight, concurrent add/promotion/removal, lost marker response, leader loss plus membership change plus re-election, and member restart; assert old-term proposals are rejected before Raft append, marker/cutoff identity, and each member persisted-index certificate before Drained.", "stop the original checkpoint donor permanently while a syncing voter is in PrepareSnapshot, keep another checkpointable voter alive, force the syncing voter to win on its log, retain the serving gate until restore completes, then compare literal business balances on both survivors and acknowledge a new write (tests/e2e/cluster/checkpoint_donor_loss_test.go); separately lose a donor during PrepareSnapshot and after partial FetchFile bytes reach disk, require complete-session fallback, exact attempt counts, staging and progress reset, correct final bytes, and temporary-file cleanup (internal/application/ctrl/snapshot_fetcher_donor_loss_test.go)", - "inject a successful snapshot-file write followed by a failed WAL snapshot-record append, run an idle maintenance retry, and assert that the previous snapshot remains authoritative until the record is durable and that no required WAL prefix is compacted (internal/storage/wal/wal_default_test.go and internal/infra/node/maintenance_test.go)" + "inject a successful snapshot-file write followed by a failed WAL snapshot-record append, run an idle maintenance retry, and assert that the previous snapshot remains authoritative until the record is durable and that no required WAL prefix is compacted (internal/storage/wal/wal_default_test.go and internal/infra/node/maintenance_test.go)", + "Run a follower-first rolling upgrade against a peer that returns Unimplemented for RefreshMembership; assert readiness opens with conservative evidence, retries continue, and a later compatible peer commits the refresh. Then exercise capability declaration/activation and membership ConfChange in the same or adjacent FSM batch, recording committed policy, evidence, ConfState, and final admission decisions on every replica." ] } diff --git a/docs/technical/audits/raft-membership-leadership.md b/docs/technical/audits/raft-membership-leadership.md index 5779f15f59..b77bc7da1e 100644 --- a/docs/technical/audits/raft-membership-leadership.md +++ b/docs/technical/audits/raft-membership-leadership.md @@ -75,6 +75,23 @@ question rather than inventing a membership guarantee. ## Transition-specific evidence +### Capability evidence and policy activation + +For bootstrap join, restart refresh, promotion, and membership changes, record +the member's `(node_id, instance_id)`, advertised capability IDs, committed +membership row, and the policy revision and active capabilities on every +replica. A restarted member must refresh its committed evidence using peers +recovered from membership even when no `--join` flag is present. Distinguish a +transient old-peer `Unimplemented` refresh response from permanent identity or +policy rejection, and prove eventual convergence through a compatible peer. +Before activating a capability, require support by the local binary and +committed evidence from every member; after activation, reject announcements +and proposals that lack the required evidence. Exercise policy activation and +membership `ConfChange` in adjacent FSM batches and compare the final policy, +membership, `ConfState`, and admission decision on every replica. This records +the foundation for future activations; the current protocol cutover uses a +coordinated stop and restart. + ### Normal membership changes Prove which `ConfChangeV2` was admitted and which configuration commits it. diff --git a/internal/adapter/grpc/auth_interceptor_test.go b/internal/adapter/grpc/auth_interceptor_test.go index dd2eb30a2c..0a4b2faacb 100644 --- a/internal/adapter/grpc/auth_interceptor_test.go +++ b/internal/adapter/grpc/auth_interceptor_test.go @@ -532,6 +532,7 @@ func newSignatureAdmission(t *testing.T, keyID string, publicKey ed25519.PublicK store, logger, nil, + nil, plan.NewBuilder(node.NewIndexTracker(1), cache, attrs, store, nil, logger, 0), meterProvider, allowWriteGate{}, diff --git a/internal/adapter/grpc/oidc_caller_attribution_test.go b/internal/adapter/grpc/oidc_caller_attribution_test.go index 497a160952..7c98399031 100644 --- a/internal/adapter/grpc/oidc_caller_attribution_test.go +++ b/internal/adapter/grpc/oidc_caller_attribution_test.go @@ -197,7 +197,7 @@ func newCallerAttributionAdmission(t *testing.T, expectedProposals int) (*admiss return result, nil }).Times(expectedProposals) - admission := admissionapp.NewAdmission(store, logger, proposer, + admission := admissionapp.NewAdmission(store, logger, proposer, nil, plan.NewBuilder(node.NewIndexTracker(1), testCache, attrs, store, nil, logger, 0), provider, allowWriteGate{}, keystore.NewKeyStore(), state.NewSharedState(), attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) diff --git a/internal/adapter/grpc/server_bootstrap.go b/internal/adapter/grpc/server_bootstrap.go index 924a34d936..ed01c0a964 100644 --- a/internal/adapter/grpc/server_bootstrap.go +++ b/internal/adapter/grpc/server_bootstrap.go @@ -15,6 +15,7 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" "github.com/formancehq/ledger/v3/internal/application/membership" + raftmembership "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/node" "github.com/formancehq/ledger/v3/internal/proto/clusterbootstrappb" "github.com/formancehq/ledger/v3/internal/proto/commonpb" @@ -133,6 +134,7 @@ func (impl *ClusterBootstrapServiceServerImpl) GetPeers(ctx context.Context, req RaftAddress: p.RaftAddress, ServiceAddress: p.ServiceAddress, InstanceId: p.InstanceID, + Capabilities: p.Capabilities, }) } @@ -222,7 +224,7 @@ func (impl *ClusterBootstrapServiceServerImpl) JoinAsLearner(ctx context.Context req.GetNodeId(), req.GetInstanceId()) } - if err := impl.membership.JoinAsLearner(ctx, req.GetNodeId(), req.GetRaftAddress(), req.GetServiceAddress(), req.GetInstanceId()); err != nil { + if err := impl.membership.JoinAsLearner(ctx, req.GetNodeId(), req.GetRaftAddress(), req.GetServiceAddress(), req.GetInstanceId(), req.GetCapabilities()); err != nil { // EN-1436: a JoinAsLearner call reaches the leader only when the caller // has no CLUSTER_JOINED marker on its WAL (see bootstrap/module.go // tryAddLearner). If the leader's Progress already carries this @@ -293,6 +295,52 @@ func (impl *ClusterBootstrapServiceServerImpl) JoinAsLearner(ctx context.Context return &clusterbootstrappb.JoinAsLearnerResponse{}, nil } +// RefreshMembership forwards a normal-restart membership refresh to the +// leader. Unlike JoinAsLearner, this endpoint only accepts an incarnation that +// is already present in the committed membership and proposes UpdateNode. +func (impl *ClusterBootstrapServiceServerImpl) RefreshMembership(ctx context.Context, req *clusterbootstrappb.RefreshMembershipRequest) (*clusterbootstrappb.RefreshMembershipResponse, error) { + if err := impl.checkClusterID(ctx); err != nil { + return nil, err + } + + if err := raftmembership.ValidateInstanceID(req.GetInstanceId()); err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + if !impl.node.IsLeader() { + conn, err := impl.leaderRaftConn() + if err != nil { + return nil, convertToGRPCErrorWithContext(ctx, err, impl.logger) + } + + outCtx := ctx + if impl.clusterID != "" { + outCtx = metadata.AppendToOutgoingContext(ctx, node.MetadataKeyClusterID, impl.clusterID) + } + + resp, err := clusterbootstrappb.NewClusterBootstrapServiceClient(conn).RefreshMembership(outCtx, req) + if err != nil { + return nil, forwardedCallError(err) + } + + return resp, nil + } + + removed, err := impl.membership.IsRemoved(req.GetNodeId(), req.GetInstanceId()) + if err != nil { + return nil, status.Error(codes.Internal, "reading removed-member registry") + } + if removed { + return nil, status.Errorf(codes.FailedPrecondition, "node %d (instance %x) was previously removed from this cluster", req.GetNodeId(), req.GetInstanceId()) + } + + if err := impl.membership.RefreshMembership(ctx, req.GetNodeId(), req.GetRaftAddress(), req.GetServiceAddress(), req.GetInstanceId(), req.GetCapabilities()); err != nil { + return nil, convertToGRPCErrorWithContext(ctx, err, impl.logger) + } + + return &clusterbootstrappb.RefreshMembershipResponse{}, nil +} + // forwardedCallError maps the failure of a call forwarded to the leader. // The leader converts its own context errors to Unavailable, so Canceled // and DeadlineExceeded can only come from the forwarding hop: the shared diff --git a/internal/adapter/grpc/server_cluster.go b/internal/adapter/grpc/server_cluster.go index e917c5f1bc..76a3e58a5b 100644 --- a/internal/adapter/grpc/server_cluster.go +++ b/internal/adapter/grpc/server_cluster.go @@ -500,7 +500,7 @@ func (impl *ClusterServiceServerImpl) AddLearner(ctx context.Context, req *clust // Administrative registration is allowed only for a concrete peer // incarnation. The caller must read the target's persisted INSTANCE_ID; // creating an identity-less phantom member is forbidden. - if err := impl.membership.AddLearner(ctx, req.GetNodeId(), req.GetRaftAddress(), req.GetServiceAddress(), req.GetInstanceId()); err != nil { + if err := impl.membership.AddLearner(ctx, req.GetNodeId(), req.GetRaftAddress(), req.GetServiceAddress(), req.GetInstanceId(), req.GetCapabilities()); err != nil { return nil, err } diff --git a/internal/application/admission/admission.go b/internal/application/admission/admission.go index 0fb796c564..07220c8813 100644 --- a/internal/application/admission/admission.go +++ b/internal/application/admission/admission.go @@ -26,12 +26,12 @@ import ( "github.com/formancehq/ledger/v3/internal/domain/processing/numscript" "github.com/formancehq/ledger/v3/internal/infra/attributes" "github.com/formancehq/ledger/v3/internal/infra/health" + "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/node" "github.com/formancehq/ledger/v3/internal/infra/plan" "github.com/formancehq/ledger/v3/internal/infra/state" "github.com/formancehq/ledger/v3/internal/pkg/commands" "github.com/formancehq/ledger/v3/internal/pkg/futures" - "github.com/formancehq/ledger/v3/internal/pkg/vtmarshal" "github.com/formancehq/ledger/v3/internal/proto/commonpb" "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" "github.com/formancehq/ledger/v3/internal/proto/servicepb" @@ -51,16 +51,17 @@ type Proposer interface { // Admission handles the admission of orders into the Raft cluster. // It is responsible for preloading volumes and proposing commands. type Admission struct { - store *dal.Store - logger logging.Logger - proposer Proposer - writeGate health.WriteGate - keyStore *keystore.KeyStore - sharedState *state.SharedState - builder *plan.Builder - attrs *attributes.Attributes - numscriptCache *numscript.NumscriptCache - waitLeaderReady func(context.Context) error + store *dal.Store + logger logging.Logger + proposer Proposer + membershipCoordinator *membership.Coordinator + writeGate health.WriteGate + keyStore *keystore.KeyStore + sharedState *state.SharedState + builder *plan.Builder + attrs *attributes.Attributes + numscriptCache *numscript.NumscriptCache + waitLeaderReady func(context.Context) error // auditProjectionState is node-local admission state. It may reject a // checkpoint proposal before Raft, but never affects deterministic apply. auditProjectionState func() (disabled, rebuilding bool) @@ -139,6 +140,7 @@ func NewAdmission( store *dal.Store, logger logging.Logger, proposer Proposer, + membershipCoordinator *membership.Coordinator, builder *plan.Builder, meterProvider metric.MeterProvider, writeGate health.WriteGate, @@ -150,16 +152,17 @@ func NewAdmission( opts ...func(*Admission), ) *Admission { a := &Admission{ - store: store, - logger: logger, - proposer: proposer, - builder: builder, - writeGate: writeGate, - keyStore: keyStore, - sharedState: sharedState, - attrs: attrs, - numscriptCache: numscriptCache, - waitLeaderReady: waitLeaderReady, + store: store, + logger: logger, + proposer: proposer, + membershipCoordinator: membershipCoordinator, + builder: builder, + writeGate: writeGate, + keyStore: keyStore, + sharedState: sharedState, + attrs: attrs, + numscriptCache: numscriptCache, + waitLeaderReady: waitLeaderReady, } for _, opt := range opts { opt(a) @@ -488,7 +491,7 @@ func (a *Admission) recordPhaseOnExit(ctx context.Context, hist metric.Float64Hi func (a *Admission) Admit(ctx context.Context, req *servicepb.ApplyRequest) (*domain.ApplyResult, error) { return a.admit(ctx, func(ctx context.Context, caller *commonpb.CallerSnapshot) (*preparedAdmission, error) { return a.prepareExecutionBatch(ctx, caller, req) - }) + }, a.proposeAdmission) } // AdmitClusterPolicy admits the leader's internal cluster-policy reconciliation @@ -500,7 +503,7 @@ func (a *Admission) AdmitClusterPolicy(ctx context.Context, policy *commonpb.Clu return a.admit(ctx, func(ctx context.Context, caller *commonpb.CallerSnapshot) (*preparedAdmission, error) { return a.prepareClusterPolicy(ctx, caller, policy) - }) + }, a.proposeClusterPolicy) } type preparedAdmission struct { @@ -549,7 +552,9 @@ func (a *Admission) prepareClusterPolicy(ctx context.Context, caller *commonpb.C return &preparedAdmission{command: command, build: build}, nil } -func (a *Admission) admit(ctx context.Context, prepare func(context.Context, *commonpb.CallerSnapshot) (*preparedAdmission, error)) (response *domain.ApplyResult, err error) { +type admissionPropose func(context.Context, *raftcmdpb.Proposal, *plan.BuildResult) (*futures.Future[state.ApplyResult], func(), error) + +func (a *Admission) admit(ctx context.Context, prepare func(context.Context, *commonpb.CallerSnapshot) (*preparedAdmission, error), propose admissionPropose) (response *domain.ApplyResult, err error) { caller, err := auth.ResolveCallerAttribution(ctx) if err != nil { invalid, ok := errors.AsType[*domain.ErrInvalidCallerAttribution](err) @@ -586,7 +591,7 @@ func (a *Admission) admit(ctx context.Context, prepare func(context.Context, *co return nil, err } - fsmFuture, cleanup, err := a.proposeAdmission(ctx, prepared.command, prepared.build) + fsmFuture, cleanup, err := propose(ctx, prepared.command, prepared.build) if err != nil { return nil, err } @@ -595,6 +600,21 @@ func (a *Admission) admit(ctx context.Context, prepare func(context.Context, *co return a.resolveAdmissionResult(ctx, prepared.command, fsmFuture) } +func (a *Admission) proposeClusterPolicy(ctx context.Context, command *raftcmdpb.Proposal, build *plan.BuildResult) (*futures.Future[state.ApplyResult], func(), error) { + // Fence membership before the proposal guard locks the index tracker. + if a.membershipCoordinator == nil { + return nil, nil, errors.New("cluster policy admission has no membership coordinator") + } + release, err := a.membershipCoordinator.AcquireClusterPolicy(ctx) + if err != nil { + return nil, nil, fmt.Errorf("fencing membership for cluster policy: %w", err) + } + future, cleanup, err := a.proposeAdmission(ctx, command, build) + release(future) + + return future, cleanup, err +} + func (a *Admission) prepareExecutionBatchAdmission(ctx context.Context, req *servicepb.ApplyRequest) (verifiedBatch, error) { batch, err := a.resolveExecutionBatch(ctx, req) if err != nil { @@ -780,18 +800,21 @@ func (a *Admission) proposeAdmission(ctx context.Context, cmd *raftcmdpb.Proposa // Per-order coverage bits depend on the final AttributeCoverage slice // (positions in cmd.ExecutionPlan.Attributes), and AcquireProposalGuard may // swap cmd.ExecutionPlan for a rebuilt ExecutionPlan on a generation shift. - // Compute the bits inside marshalFn so every (re-)marshal sees the - // current Preload — the runner calls marshalFn again after the + // Compute the bits before final proposal construction so every rebuild sees the + // current Preload — the runner calls finalizeFn again after the // rebuild, keeping bits and plans in sync. ctx, proposeSpan := tracer.Start(ctx, "admission.propose") runResult, err := a.builder.Run( ctx, cmd, build, - func(c *raftcmdpb.Proposal) ([]byte, error) { + func(c *raftcmdpb.Proposal) error { // Coverage and productions are already assigned by the - // runner before this callback runs. We just marshal + emit - // admission metrics. - return a.marshalCommand(ctx, c) + // runner before this callback runs. The node proposal factory + // performs the canonical serialization; this callback only + // records the final command size. + a.commandSizeHistogram.Record(ctx, int64(c.SizeVT())) + + return nil }, a.proposer, ) @@ -860,7 +883,9 @@ func (a *Admission) proposeAdmission(ctx context.Context, cmd *raftcmdpb.Proposa guard.ReleaseLoaders() a.proposeQueueInflight.Add(-1) - return nil, nil, err + // The proposal was already enqueued. Keep its completion handle so + // capability activation retains the membership fence after cancellation. + return fsmFuture, nil, err } recordProposeDuration() @@ -996,13 +1021,14 @@ func (a *Admission) Barrier(ctx context.Context) (uint64, error) { } cmd := commands.NewCommand() // no orders = no-op - proposalData, err := a.marshalCommand(ctx, cmd) + // The proposal factory stamps the semantic capability declaration and + // serializes this final command atomically. + a.commandSizeHistogram.Record(ctx, int64(cmd.SizeVT())) + proposal, err := node.NewProposal(cmd.GetId(), cmd) if err != nil { return 0, fmt.Errorf("marshaling barrier command: %w", err) } - proposal := node.NewProposal(cmd.GetId(), proposalData) - // Lock the tracker to serialize the Increment with guarded proposals, // preventing preload boundary mismatches in the FSM. a.builder.LockTracker() @@ -1035,23 +1061,6 @@ func (a *Admission) Barrier(ctx context.Context) (uint64, error) { return result.AppliedIndex, nil } -// marshalCommand marshals a proposal command into a newly allocated byte slice -// using a pooled buffer. The returned slice is safe for Raft retention. -func (a *Admission) marshalCommand(ctx context.Context, cmd *raftcmdpb.Proposal) ([]byte, error) { - _, marshalSpan := tracer.Start(ctx, "admission.marshal") - defer marshalSpan.End() - - proposalData, err := vtmarshal.MarshalCopy(cmd) - if err != nil { - return nil, fmt.Errorf("marshaling command: %w", err) - } - - a.commandSizeHistogram.Record(ctx, int64(len(proposalData))) - marshalSpan.SetAttributes(attribute.Int("command.size_bytes", len(proposalData))) - - return proposalData, nil -} - // verifiedBatch is the trusted content of an ApplyRequest after signature // verification: the ordered requests, the batch idempotency key, and the // signing envelope (nil if unsigned), propagated onto the Proposal for audit. diff --git a/internal/application/admission/admission_phase_metrics_test.go b/internal/application/admission/admission_phase_metrics_test.go index 6f2c1c1287..a8220889e6 100644 --- a/internal/application/admission/admission_phase_metrics_test.go +++ b/internal/application/admission/admission_phase_metrics_test.go @@ -17,6 +17,7 @@ import ( "github.com/formancehq/ledger/v3/internal/infra/attributes" "github.com/formancehq/ledger/v3/internal/infra/cache" "github.com/formancehq/ledger/v3/internal/infra/health" + "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/node" "github.com/formancehq/ledger/v3/internal/infra/plan" "github.com/formancehq/ledger/v3/internal/infra/state" @@ -59,6 +60,7 @@ func createTestAdmissionWithReader(t *testing.T, store *dal.Store, proposer Prop store, logger, proposer, + &membership.Coordinator{}, testPreloader, provider, writeGate, diff --git a/internal/application/admission/admission_test.go b/internal/application/admission/admission_test.go index 3f8d3f18be..b829872b99 100644 --- a/internal/application/admission/admission_test.go +++ b/internal/application/admission/admission_test.go @@ -118,6 +118,7 @@ func createTestAdmission(t *testing.T, store *dal.Store) (*Admission, *attribute store, logger, nil, // no proposer needed for unit tests + nil, // no membership fence needed for unit tests testPreloader, noop.NewMeterProvider(), nil, // no health checker needed for unit tests diff --git a/internal/application/admission/capability_policy_test.go b/internal/application/admission/capability_policy_test.go new file mode 100644 index 0000000000..96a993502b --- /dev/null +++ b/internal/application/admission/capability_policy_test.go @@ -0,0 +1,40 @@ +package admission + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/formancehq/ledger/v3/internal/infra/health" + "github.com/formancehq/ledger/v3/internal/infra/node" + "github.com/formancehq/ledger/v3/internal/infra/state" + "github.com/formancehq/ledger/v3/internal/pkg/futures" +) + +type barrierTestProposer struct{} + +func (barrierTestProposer) InitialIndex() uint64 { return 1 } + +func (barrierTestProposer) Propose(_ context.Context, proposal *node.Proposal) (*futures.Future[state.ApplyResult], error) { + proposal.Resolve(nil, nil) + future := futures.New[state.ApplyResult]() + future.Resolve(state.ApplyResult{AppliedIndex: 1}, nil) + + return future, nil +} + +type allowWrites struct{} + +func (allowWrites) CheckWritesAllowed() error { return nil } + +var _ health.WriteGate = allowWrites{} + +func TestBarrierStampsSemanticCapabilities(t *testing.T) { + a, _ := createTestAdmission(t, createTestStore(t)) + a.writeGate = allowWrites{} + a.proposer = barrierTestProposer{} + + _, err := a.Barrier(context.Background()) + require.NoError(t, err) +} diff --git a/internal/application/admission/cluster_policy_gate_test.go b/internal/application/admission/cluster_policy_gate_test.go index 03d4451beb..ebdf789c65 100644 --- a/internal/application/admission/cluster_policy_gate_test.go +++ b/internal/application/admission/cluster_policy_gate_test.go @@ -185,3 +185,19 @@ func TestAdmit_ClusterPolicyWriteReadinessGate(t *testing.T) { require.ErrorIs(t, err, ErrMaintenanceMode) }) } + +func TestAdmitClusterPolicyFencesMembershipWithoutActivations(t *testing.T) { + t.Parallel() + + store := createTestStoreWithoutPolicy(t) + proposer := NewMockProposer(gomock.NewController(t)) + a, _ := createTestAdmissionWithReader(t, store, proposer) + a.membershipCoordinator.Lock() + defer a.membershipCoordinator.Unlock() + + ctx, cancel := context.WithTimeout(attributedTestContext(context.Background()), 100*stdtime.Millisecond) + defer cancel() + + _, err := a.AdmitClusterPolicy(ctx, &commonpb.ClusterPolicy{Revision: 1, QueryCheckpointLimit: 10}) + require.ErrorIs(t, err, context.DeadlineExceeded) +} diff --git a/internal/application/admission/metadata_replay_test.go b/internal/application/admission/metadata_replay_test.go index 17cdb98307..b0730556ad 100644 --- a/internal/application/admission/metadata_replay_test.go +++ b/internal/application/admission/metadata_replay_test.go @@ -21,6 +21,7 @@ import ( "github.com/formancehq/ledger/v3/internal/infra/attributes" "github.com/formancehq/ledger/v3/internal/infra/cache" "github.com/formancehq/ledger/v3/internal/infra/health" + "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/node" "github.com/formancehq/ledger/v3/internal/infra/plan" "github.com/formancehq/ledger/v3/internal/infra/state" @@ -72,7 +73,7 @@ func newMetadataReplayHarness(t testing.TB) *metadataReplayHarness { registry := state.NewStateRegistry(c, attrs) ks := keystore.NewKeyStore() shared := state.NewSharedState() - machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meter, ks, shared, signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meter, ks, shared, signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, state.NewRecovery(machine, store).RecoverState()) tracker := node.NewIndexTracker(1) @@ -117,7 +118,7 @@ func newMetadataReplayHarness(t testing.TB) *metadataReplayHarness { }).AnyTimes() gate := health.NewMockWriteGate(gomock.NewController(t)) gate.EXPECT().CheckWritesAllowed().Return(nil).AnyTimes() - admission := NewAdmission(store, logger, proposer, plan.NewBuilder(tracker, c, attrs, store, nil, logger, 0), meter, gate, ks, shared, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) + admission := NewAdmission(store, logger, proposer, &membership.Coordinator{}, plan.NewBuilder(tracker, c, attrs, store, nil, logger, 0), meter, gate, ks, shared, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) h.admission = admission h.apply = func(p *raftcmdpb.Proposal) state.ApplyResult { p.CallerSnapshot = commands.SystemCallerSnapshot(commands.ComponentClusterPolicy) diff --git a/internal/application/check/checker_fsm_purge_test.go b/internal/application/check/checker_fsm_purge_test.go index 09adb835c4..7893dbd5cf 100644 --- a/internal/application/check/checker_fsm_purge_test.go +++ b/internal/application/check/checker_fsm_purge_test.go @@ -70,6 +70,7 @@ func newFSMPurgeMachine(t *testing.T) (*state.Machine, *dal.Store, *attributes.A fsmPurgeAuditKey, 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, + state.NoopCapabilityPolicyCommitHandler, ) require.NoError(t, err) diff --git a/internal/application/check/checker_fsm_tombstone_test.go b/internal/application/check/checker_fsm_tombstone_test.go index 583587ddd1..ff2ca0dc2f 100644 --- a/internal/application/check/checker_fsm_tombstone_test.go +++ b/internal/application/check/checker_fsm_tombstone_test.go @@ -67,6 +67,7 @@ func newTombstoneFSMMachine(t *testing.T) (*state.Machine, *dal.Store, *attribut tombstoneFSMAuditKey, 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, + state.NoopCapabilityPolicyCommitHandler, ) require.NoError(t, err) diff --git a/internal/application/membership/membership.go b/internal/application/membership/membership.go index 5aec056067..005ba94923 100644 --- a/internal/application/membership/membership.go +++ b/internal/application/membership/membership.go @@ -50,6 +50,7 @@ type Peer struct { RaftAddress string ServiceAddress string InstanceID []byte + Capabilities []uint32 } // Service is the single owner of cluster membership operations. @@ -89,19 +90,37 @@ func (s *Service) IsRemoved(nodeID uint64, instanceID []byte) (bool, error) { // // Returns node.ErrNodeAlreadyInCluster on idempotent retries; adapters // map that to their transport-specific "already-exists" status. -func (s *Service) AddLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte) error { - return s.addLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, false) +// AddLearner registers a learner and carries the joining binary's capability +// evidence in the committed ConfChange. +func (s *Service) AddLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { + return s.addLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs, false) } // JoinAsLearner registers a node that is booting with an empty WAL and no -// CLUSTER_JOINED marker. Keeping this entry point distinct from the admin -// AddLearner operation preserves EN-1436's stale-progress fail-fast without -// inferring caller intent from a missing instanceID sentinel. -func (s *Service) JoinAsLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte) error { - return s.addLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, true) +// CLUSTER_JOINED marker, carrying the joining binary's capability evidence +// through the committed ConfChange. Keeping this entry point distinct from the +// admin AddLearner operation preserves EN-1436's stale-progress fail-fast +// without inferring caller intent from a missing instanceID sentinel. +func (s *Service) JoinAsLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { + return s.addLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs, true) } -func (s *Service) addLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, bootJoin bool) error { +// RefreshMembership updates the durable identity, endpoints and capability +// evidence of an existing member. The node must already be present in the +// committed Raft membership; the leader commits the update through Raft. +func (s *Service) RefreshMembership(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { + if err := validatePeer(nodeID, raftAddr, serviceAddr, instanceID); err != nil { + return err + } + + if err := s.node.RefreshMembership(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs); err != nil { + return fmt.Errorf("refreshing membership: %w", err) + } + + return nil +} + +func (s *Service) addLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32, bootJoin bool) error { if err := validatePeer(nodeID, raftAddr, serviceAddr, instanceID); err != nil { return err } @@ -119,9 +138,9 @@ func (s *Service) addLearner(ctx context.Context, nodeID uint64, raftAddr, servi var err error if bootJoin { - err = s.node.JoinAsLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID) + err = s.node.JoinAsLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs) } else { - err = s.node.AddLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID) + err = s.node.AddLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs) } if err != nil { return fmt.Errorf("adding learner: %w", err) @@ -199,6 +218,7 @@ func (s *Service) ListPeers(ctx context.Context) ([]Peer, error) { RaftAddress: raftAddr, ServiceAddress: serviceAddr, InstanceID: peer.InstanceID, + Capabilities: append([]uint32(nil), peer.Capabilities...), }) } diff --git a/internal/application/membership/membership_node_test.go b/internal/application/membership/membership_node_test.go index 5525329b61..13d29320c3 100644 --- a/internal/application/membership/membership_node_test.go +++ b/internal/application/membership/membership_node_test.go @@ -72,7 +72,7 @@ func newMembershipServiceNodeWithPool(t *testing.T) (*Service, *transport.Connec } }) m, err := inframembership.NewMembership(inframembership.NewPeerStore(store), raftTransport, pool, - 1, "self:7777", "self:8888", []byte("self-instance-id"), logger) + 1, "self:7777", "self:8888", []byte("self-instance-id"), logger, inframembership.NoopClusterPolicyProvider) require.NoError(t, err) c, err := cache.New(1000, nil) @@ -80,7 +80,9 @@ func newMembershipServiceNodeWithPool(t *testing.T) (*Service, *transport.Connec registry := state.NewStateRegistry(c, attributes.New()) fsm, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, nil, state.NewSharedState(), - signal.NewNotifications(), nil, "membership-service-test", 0, m.WriteConfChange) + signal.NewNotifications(), nil, "membership-service-test", 0, + func(entry *raftpb.Entry, session *dal.WriteSession) error { return m.WriteConfChange(entry, session) }, + state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) recovery := state.NewRecovery(fsm, store) require.NoError(t, recovery.RecoverState()) @@ -142,34 +144,35 @@ func TestService_LearnerRegistrationCommitsAndPreservesCallerIntent(t *testing.T } t.Run(name, func(t *testing.T) { t.Parallel() - synctest.Test(t, func(t *testing.T) { - s := newMembershipServiceNode(t) - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() - register := s.AddLearner + s := newMembershipServiceNode(t) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + register := func(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilities []uint32) error { if bootJoin { - register = s.JoinAsLearner + return s.JoinAsLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilities) } - require.NoError(t, register(ctx, 2, "peer:7777", "peer:8888", []byte("peer-instance-id"))) - peers, err := s.ListPeers(ctx) - require.NoError(t, err) - require.Contains(t, peers, Peer{ - ID: 2, RaftAddress: "peer:7777", ServiceAddress: "peer:8888", InstanceID: []byte("peer-instance-id"), - }) - - // The existing voter has replicated its own no-op. An administrative - // retry is idempotent, while a fresh-WAL boot against that progress - // must report stale state, even with the same persisted identity. - before := s.infraMembership.PeerAddresses() - err = register(ctx, 1, "replacement:7777", "replacement:8888", []byte("self-instance-id")) - if bootJoin { - require.ErrorIs(t, err, node.ErrNodeStaleProgress) - } else { - require.ErrorIs(t, err, node.ErrNodeAlreadyInCluster) - } - require.ErrorContains(t, err, "adding learner:") - require.Equal(t, before, s.infraMembership.PeerAddresses(), "rejected registration must retain committed addresses") + + return s.AddLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilities) + } + require.NoError(t, register(ctx, 2, "peer:7777", "peer:8888", []byte("peer-instance-id"), nil)) + peers, err := s.ListPeers(ctx) + require.NoError(t, err) + require.Contains(t, peers, Peer{ + ID: 2, RaftAddress: "peer:7777", ServiceAddress: "peer:8888", InstanceID: []byte("peer-instance-id"), }) + + // The existing voter has replicated its own no-op. An administrative + // retry is idempotent, while a fresh-WAL boot against that progress + // must report stale state, even with the same persisted identity. + before := s.infraMembership.PeerAddresses() + err = register(ctx, 1, "replacement:7777", "replacement:8888", []byte("self-instance-id"), nil) + if bootJoin { + require.ErrorIs(t, err, node.ErrNodeStaleProgress) + } else { + require.ErrorIs(t, err, node.ErrNodeAlreadyInCluster) + } + require.ErrorContains(t, err, "adding learner:") + require.Equal(t, before, s.infraMembership.PeerAddresses(), "rejected registration must retain committed addresses") }) } } @@ -181,9 +184,9 @@ func TestService_ListPeersReturnsCompleteIndependentSnapshot(t *testing.T) { s := newMembershipServiceNode(t) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() - require.NoError(t, s.AddLearner(ctx, 2, "peer:7777", "peer:8888", []byte("peer-instance-id"))) + require.NoError(t, s.AddLearner(ctx, 2, "peer:7777", "peer:8888", []byte("peer-instance-id"), nil)) // A cached row alone must not leak into the configured member response. - require.NoError(t, s.infraMembership.Set(3, "extra:7777", "extra:8888", []byte("extra-instanceid"))) + require.NoError(t, s.infraMembership.Set(3, "extra:7777", "extra:8888", []byte("extra-instanceid"), nil)) want := []Peer{ {ID: 1, RaftAddress: "public-self:7777", ServiceAddress: "public-self:8888", InstanceID: []byte("self-instance-id")}, {ID: 2, RaftAddress: "peer:7777", ServiceAddress: "peer:8888", InstanceID: []byte("peer-instance-id")}, @@ -199,3 +202,35 @@ func TestService_ListPeersReturnsCompleteIndependentSnapshot(t *testing.T) { require.ElementsMatch(t, want, fresh, "caller mutation must not corrupt a later discovery response") }) } + +func TestService_JoinAsLearnerPersistsEvidence(t *testing.T) { + t.Parallel() + + s := newMembershipServiceNode(t) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + require.NoError(t, s.JoinAsLearner(ctx, 2, "peer:7777", "peer:8888", []byte("peer-instance-id"), []uint32{7, 9})) + + peer := s.infraMembership.PeerAddresses()[2] + require.Equal(t, []uint32{7, 9}, peer.Capabilities) + peers, err := s.ListPeers(ctx) + require.NoError(t, err) + require.Contains(t, peers, Peer{ + ID: 2, RaftAddress: "peer:7777", ServiceAddress: "peer:8888", + InstanceID: []byte("peer-instance-id"), Capabilities: []uint32{7, 9}, + }) +} + +func TestService_AddLearnerWithCapabilitiesPersistsEvidence(t *testing.T) { + t.Parallel() + + s := newMembershipServiceNode(t) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + require.NoError(t, s.AddLearner(ctx, 2, "peer:7777", "peer:8888", []byte("peer-instance-id"), []uint32{7, 9})) + + peer := s.infraMembership.PeerAddresses()[2] + require.Equal(t, []uint32{7, 9}, peer.Capabilities) +} diff --git a/internal/application/membership/membership_test.go b/internal/application/membership/membership_test.go index 4cbba884a2..f180f0d1ab 100644 --- a/internal/application/membership/membership_test.go +++ b/internal/application/membership/membership_test.go @@ -41,7 +41,7 @@ func TestService_AddLearner_ValidatesRequest(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Parallel() - err := s.AddLearner(context.Background(), tc.nodeID, tc.raftAddr, tc.serviceAddr, tc.instanceID) + err := s.AddLearner(context.Background(), tc.nodeID, tc.raftAddr, tc.serviceAddr, tc.instanceID, nil) require.Error(t, err) assert.Contains(t, err.Error(), tc.wantSubstr) }) @@ -58,7 +58,7 @@ func TestService_AddLearner_RejectionPreservesServiceRouting(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() - err := s.AddLearner(ctx, 1, "self:7777", "attacker:8080", []byte("self-instance-id")) + err := s.AddLearner(ctx, 1, "self:7777", "attacker:8080", []byte("self-instance-id"), nil) require.ErrorIs(t, err, node.ErrNodeAlreadyInCluster) require.Equal(t, committedAddr, servicePool.GetPeerAddress(1)) }) @@ -67,7 +67,7 @@ func TestService_AddLearner_RejectionPreservesServiceRouting(t *testing.T) { func TestService_JoinAsLearner_RejectsMissingInstanceID(t *testing.T) { t.Parallel() - err := (&Service{}).JoinAsLearner(context.Background(), 1, "r:1", "s:1", nil) + err := (&Service{}).JoinAsLearner(context.Background(), 1, "r:1", "s:1", nil, nil) require.ErrorContains(t, err, "instance_id") } diff --git a/internal/application/mirror/worker.go b/internal/application/mirror/worker.go index 9480ae279f..01c1565030 100644 --- a/internal/application/mirror/worker.go +++ b/internal/application/mirror/worker.go @@ -18,7 +18,6 @@ import ( "github.com/formancehq/ledger/v3/internal/infra/plan" "github.com/formancehq/ledger/v3/internal/pkg/commands" "github.com/formancehq/ledger/v3/internal/pkg/signal" - "github.com/formancehq/ledger/v3/internal/pkg/vtmarshal" "github.com/formancehq/ledger/v3/internal/pkg/worker" "github.com/formancehq/ledger/v3/internal/proto/commonpb" "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" @@ -555,18 +554,13 @@ func (w *Worker) processBatch(ctx context.Context) (bool, error) { // Run preload + propose via the shared runner. Mirror is a // single-shot caller (no concurrent admissions sharing loaders), // so we release loaders immediately after the runner returns. - marshalFn := func(c *raftcmdpb.Proposal) ([]byte, error) { - data, err := marshalMirrorCommand(c) - if err != nil { - return nil, err - } - - w.commandSize.Record(ctx, int64(len(data)), attrs) + finalizeFn := func(c *raftcmdpb.Proposal) error { + w.commandSize.Record(ctx, int64(c.SizeVT()), attrs) - return data, nil + return nil } - runResult, err := w.builder.Run(ctx, cmd, build, marshalFn, w.proposer) + runResult, err := w.builder.Run(ctx, cmd, build, finalizeFn, w.proposer) if err != nil { return false, err } @@ -780,7 +774,7 @@ func (w *Worker) proposeMirrorSync(ctx context.Context, update *raftcmdpb.Mirror return false } - runResult, err := w.builder.Run(ctx, cmd, build, marshalMirrorCommand, w.proposer) + runResult, err := w.builder.Run(ctx, cmd, build, func(*raftcmdpb.Proposal) error { return nil }, w.proposer) if err != nil { logFailure("could not be proposed", err) @@ -814,12 +808,6 @@ func (w *Worker) proposeMirrorSync(ctx context.Context, update *raftcmdpb.Mirror return true } -// marshalMirrorCommand marshals a proposal command into a newly allocated byte -// slice using a pooled buffer. The returned slice is safe for Raft retention. -func marshalMirrorCommand(cmd *raftcmdpb.Proposal) ([]byte, error) { - return vtmarshal.MarshalCopy(cmd) -} - // extractMirrorNeeds builds plan.Coverage from a mirror proposal's orders. // Returns the proposal-wide aggregate Coverage alongside a parallel slice with // one Coverage per order, used to compute Order.coverage_bits after diff --git a/internal/application/mirror/worker_diagnostic_test.go b/internal/application/mirror/worker_diagnostic_test.go index a4babb5a39..c31df58bcd 100644 --- a/internal/application/mirror/worker_diagnostic_test.go +++ b/internal/application/mirror/worker_diagnostic_test.go @@ -80,7 +80,7 @@ func TestWorker_MalformedURLDoesNotDisclosePassword(t *testing.T) { registry := state.NewStateRegistry(c, attrs) keys := keystore.NewKeyStore() shared := state.NewSharedState() - machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keys, shared, signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keys, shared, signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) writeMirrorMetadataPolicy(t, store, mirrorMetadataPolicy()) require.NoError(t, state.NewRecovery(machine, store).RecoverState()) @@ -89,7 +89,7 @@ func TestWorker_MalformedURLDoesNotDisclosePassword(t *testing.T) { proposer := &applyingMirrorProposer{t: t, machine: machine, store: store, tracker: tracker} writeGate := health.NewMockWriteGate(gomock.NewController(t)) writeGate.EXPECT().CheckWritesAllowed().Return(nil) - adm := admission.NewAdmission(store, logger, proposer, builder, meters, writeGate, keys, shared, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) + adm := admission.NewAdmission(store, logger, proposer, nil, builder, meters, writeGate, keys, shared, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) config := &commonpb.MirrorSourceConfig{ LedgerName: "source-ledger", Type: &commonpb.MirrorSourceConfig_Http{Http: &commonpb.HttpMirrorSourceConfig{ @@ -238,7 +238,7 @@ func TestWorker_TransportErrorsDoNotDiscloseCredentials(t *testing.T) { registry := state.NewStateRegistry(c, attrs) keys := keystore.NewKeyStore() shared := state.NewSharedState() - machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keys, shared, signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keys, shared, signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) writeMirrorMetadataPolicy(t, store, mirrorMetadataPolicy()) require.NoError(t, state.NewRecovery(machine, store).RecoverState()) @@ -247,7 +247,7 @@ func TestWorker_TransportErrorsDoNotDiscloseCredentials(t *testing.T) { proposer := &applyingMirrorProposer{t: t, machine: machine, store: store, tracker: tracker} writeGate := health.NewMockWriteGate(gomock.NewController(t)) writeGate.EXPECT().CheckWritesAllowed().Return(nil) - adm := admission.NewAdmission(store, logger, proposer, builder, meters, writeGate, keys, shared, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) + adm := admission.NewAdmission(store, logger, proposer, nil, builder, meters, writeGate, keys, shared, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }) sourceUser := tc.user if mode != "direct" { diff --git a/internal/application/mirror/worker_postgres_integration_test.go b/internal/application/mirror/worker_postgres_integration_test.go index e1412c54cf..2d59164fc5 100644 --- a/internal/application/mirror/worker_postgres_integration_test.go +++ b/internal/application/mirror/worker_postgres_integration_test.go @@ -65,7 +65,7 @@ func TestWorker_PostgresGapCommitAndRollback(t *testing.T) { c, err := cache.New(100, meters.Meter("test")) require.NoError(t, err) registry := state.NewStateRegistry(c, attrs) - machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keystore.NewKeyStore(), state.NewSharedState(), signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keystore.NewKeyStore(), state.NewSharedState(), signal.NewNotifications(), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) writeMirrorMetadataPolicy(t, store, mirrorMetadataPolicy()) require.NoError(t, state.NewRecovery(machine, store).RecoverState()) diff --git a/internal/application/mirror/worker_rotation_test.go b/internal/application/mirror/worker_rotation_test.go index db88ccfbb2..653357d19e 100644 --- a/internal/application/mirror/worker_rotation_test.go +++ b/internal/application/mirror/worker_rotation_test.go @@ -82,7 +82,7 @@ func TestWorker_RotationPreloadsReadOnlyLedger(t *testing.T) { require.NoError(t, err) registry := state.NewStateRegistry(c, attrs) notifier := signal.NewNotifications() - machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keystore.NewKeyStore(), state.NewSharedState(), notifier, nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keystore.NewKeyStore(), state.NewSharedState(), notifier, nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) writeMirrorMetadataPolicy(t, store, mirrorMetadataPolicy()) require.NoError(t, state.NewRecovery(machine, store).RecoverState()) diff --git a/internal/bootstrap/capability_admission_test.go b/internal/bootstrap/capability_admission_test.go new file mode 100644 index 0000000000..cecf20341c --- /dev/null +++ b/internal/bootstrap/capability_admission_test.go @@ -0,0 +1,64 @@ +package bootstrap + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/formancehq/ledger/v3/internal/infra/capabilities" + "github.com/formancehq/ledger/v3/internal/infra/node" + "github.com/formancehq/ledger/v3/internal/proto/commonpb" + "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" +) + +func TestValidateActiveRejectsIncompleteMembership(t *testing.T) { + t.Parallel() + + err := capabilities.ValidateActive( + []capabilities.CapabilityID{7}, + []capabilities.CapabilityID{7}, + []capabilities.Member{{NodeID: 1, Role: capabilities.Voter}}, + ) + + require.ErrorIs(t, err, capabilities.ErrCapabilityUnsupported) +} + +func TestValidateActiveAcceptsNoActivation(t *testing.T) { + t.Parallel() + + require.NoError(t, capabilities.ValidateActive(nil, nil, nil)) +} + +func TestValidateActiveAcceptsConvergedMembership(t *testing.T) { + t.Parallel() + + require.NoError(t, capabilities.ValidateActive( + []capabilities.CapabilityID{7}, + []capabilities.CapabilityID{7}, + []capabilities.Member{{NodeID: 1, Role: capabilities.Voter, Capabilities: []capabilities.CapabilityID{7}}}, + )) +} + +func TestProposalCapabilityAdmissionRejectsActivationBeforeRaft(t *testing.T) { + t.Parallel() + + proposal, err := node.NewProposal(1, &raftcmdpb.Proposal{ + Id: 1, + Orders: []*raftcmdpb.Order{{ + Type: &raftcmdpb.Order_SystemScoped{SystemScoped: &raftcmdpb.SystemScopedOrder{ + Payload: &raftcmdpb.SystemScopedOrder_SetClusterPolicy{SetClusterPolicy: &raftcmdpb.SetClusterPolicyOrder{ + Policy: &commonpb.ClusterPolicy{CapabilityActivations: []*commonpb.CapabilityActivation{{Id: 7, Active: true}}}, + }}, + }}, + }}, + }) + require.NoError(t, err) + + err = validateProposalCapabilityAdmission( + &commonpb.ClusterPolicy{}, + proposal, + []capabilities.Member{{NodeID: 1, Role: capabilities.Voter}}, + ) + + require.ErrorIs(t, err, capabilities.ErrCapabilityUnsupported) +} diff --git a/internal/bootstrap/cluster_policy_reconciler.go b/internal/bootstrap/cluster_policy_reconciler.go deleted file mode 100644 index ec2e046940..0000000000 --- a/internal/bootstrap/cluster_policy_reconciler.go +++ /dev/null @@ -1,52 +0,0 @@ -package bootstrap - -import ( - "context" - "time" - - "github.com/formancehq/ledger/v3/internal/pkg/worker" -) - -// clusterPolicyReconcileInterval is how often the leader re-checks the applied -// cluster policy against the desired one. Short so a fresh leader commits the -// policy — and thereby opens write readiness — promptly. -const clusterPolicyReconcileInterval = 1 * time.Second - -// ClusterPolicyReconciler periodically drives the replicated cluster policy -// toward the desired revision while this node is leader. It is periodic rather -// than one-shot on leadership acquisition so a transient proposal failure -// (propose timeout, a momentary write gate, leadership churn) self-heals on the -// next tick instead of leaving the node stuck NOT_SERVING until another -// leadership event — write readiness depends on the policy being committed -// (EN-1827). Reconciliation is idempotent: once the applied revision reaches the -// desired one, every tick is a no-op. -type ClusterPolicyReconciler struct { - reconcile func(context.Context) - interval time.Duration - w worker.Worker -} - -// NewClusterPolicyReconciler builds a reconciler that invokes reconcile on start -// and on every tick. reconcile receives a context cancelled on Stop and must -// gate on leadership itself. -func NewClusterPolicyReconciler(reconcile func(context.Context)) *ClusterPolicyReconciler { - return &ClusterPolicyReconciler{ - reconcile: reconcile, - interval: clusterPolicyReconcileInterval, - w: worker.New(), - } -} - -// Start runs an immediate reconciliation, then one per interval, until Stop. -func (r *ClusterPolicyReconciler) Start() { - r.w.Run(func(stop <-chan struct{}) { - ctx := worker.ContextFromStop(stop) - r.reconcile(ctx) - worker.RunTicker(stop, r.interval, func() { r.reconcile(ctx) }) - }) -} - -// Stop signals the background goroutine to stop and waits for it to finish. -func (r *ClusterPolicyReconciler) Stop() { - r.w.Stop() -} diff --git a/internal/bootstrap/cluster_policy_reconciler_test.go b/internal/bootstrap/cluster_policy_reconciler_test.go index da61066438..57db9a6f0b 100644 --- a/internal/bootstrap/cluster_policy_reconciler_test.go +++ b/internal/bootstrap/cluster_policy_reconciler_test.go @@ -3,10 +3,7 @@ package bootstrap import ( "context" "sync" - "sync/atomic" "testing" - "testing/synctest" - "time" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/metric/noop" @@ -20,38 +17,11 @@ import ( "github.com/formancehq/ledger/v3/internal/storage/dal" ) -// The reconciler keeps ticking until stopped, so a transient failure retries. -func TestClusterPolicyReconciler_TicksUntilStopped(t *testing.T) { - t.Parallel() - - synctest.Test(t, func(t *testing.T) { - var count atomic.Int32 - - r := NewClusterPolicyReconciler(func(context.Context) { count.Add(1) }) - r.interval = 5 * time.Millisecond - r.Start() - - // On the fake clock, two intervals yield exactly the immediate run plus - // one per tick. - time.Sleep(2 * r.interval) - synctest.Wait() - require.Equal(t, int32(3), count.Load(), "reconcile must run immediately, then on every tick") - - r.Stop() - - // Stop is synchronous (waits for the goroutine), so the count is now final. - settled := count.Load() - time.Sleep(20 * r.interval) - synctest.Wait() - require.Equal(t, settled, count.Load(), "no reconcile must run after Stop") - }) -} - type fakeAdmission struct { - mu sync.Mutex - calls int - err error - proposed *commonpb.ClusterPolicy + mu sync.Mutex + calls int + err error + lastPolicy *commonpb.ClusterPolicy } func (f *fakeAdmission) Admit(_ context.Context, _ *servicepb.ApplyRequest) (*domain.ApplyResult, error) { @@ -66,7 +36,7 @@ func (f *fakeAdmission) AdmitClusterPolicy(_ context.Context, policy *commonpb.C f.mu.Lock() defer f.mu.Unlock() f.calls++ - f.proposed = policy + f.lastPolicy = policy.CloneVT() return nil, f.err } @@ -164,8 +134,14 @@ func TestReconcileClusterPolicy(t *testing.T) { batch := store.OpenWriteSession() require.NoError(t, state.SaveClusterPolicy(batch, &commonpb.ClusterPolicy{ - Revision: cfg.ClusterPolicyRevision + 1, - QueryCheckpointLimit: cfg.QueryCheckpointLimit, + Revision: cfg.ClusterPolicyRevision + 1, + IdempotencyTtlMicros: uint64(cfg.IdempotencyTTL.Microseconds()), + QueryCheckpointLimit: cfg.QueryCheckpointLimit, + MetadataMaxEntriesPerEntity: cfg.MetadataMaxEntriesPerEntity, + MetadataMaxKeyBytes: cfg.MetadataMaxKeyBytes, + MetadataMaxValueBytes: cfg.MetadataMaxValueBytes, + MetadataMaxEntityBytes: cfg.MetadataMaxEntityBytes, + MetadataMaxCommandBytes: cfg.MetadataMaxCommandBytes, })) require.NoError(t, batch.Commit()) @@ -174,35 +150,61 @@ func TestReconcileClusterPolicy(t *testing.T) { require.Zero(t, admission.callCount(), "a newer applied revision must not be overwritten") }) -} -// Only an explicit zero TTL may reach the committed policy as 0, the -// never-expire sentinel. -func TestReconcileClusterPolicy_ProposedIdempotencyTTL(t *testing.T) { - t.Parallel() + t.Run("rejects a newer revision with a configuration conflict", func(t *testing.T) { + t.Parallel() - for _, tc := range []struct { - ttl time.Duration - wantMicros uint64 - }{ - {ttl: 0, wantMicros: 0}, - {ttl: time.Microsecond, wantMicros: 1}, - {ttl: 1500 * time.Nanosecond, wantMicros: 1}, - {ttl: 24 * time.Hour, wantMicros: uint64((24 * time.Hour).Microseconds())}, - } { - t.Run(tc.ttl.String(), func(t *testing.T) { - t.Parallel() - - cfg := validBaseConfig() - cfg.IdempotencyTTL = tc.ttl - require.NoError(t, cfg.Validate()) - - admission := &fakeAdmission{} - reconcileClusterPolicy(context.Background(), admission, newReconcilerTestStore(t), cfg, - func() bool { return true }, logging.Testing()) - - require.Equal(t, 1, admission.callCount()) - require.Equal(t, tc.wantMicros, admission.proposed.GetIdempotencyTtlMicros()) - }) - } + store := newReconcilerTestStore(t) + cfg := validBaseConfig() + batch := store.OpenWriteSession() + require.NoError(t, state.SaveClusterPolicy(batch, &commonpb.ClusterPolicy{ + Revision: cfg.ClusterPolicyRevision + 1, + QueryCheckpointLimit: cfg.QueryCheckpointLimit + 1, + })) + require.NoError(t, batch.Commit()) + + admission := &fakeAdmission{} + reconcileClusterPolicy(context.Background(), admission, store, cfg, func() bool { return true }, logging.Testing()) + + require.Zero(t, admission.callCount(), "a newer conflicting configuration must not be re-proposed") + }) + + t.Run("does not replace a same-revision configuration conflict with a capability bump", func(t *testing.T) { + t.Parallel() + + store := newReconcilerTestStore(t) + cfg := validBaseConfig() + batch := store.OpenWriteSession() + require.NoError(t, state.SaveClusterPolicy(batch, &commonpb.ClusterPolicy{ + Revision: cfg.ClusterPolicyRevision, + QueryCheckpointLimit: cfg.QueryCheckpointLimit + 1, + })) + require.NoError(t, batch.Commit()) + + admission := &fakeAdmission{} + reconcileClusterPolicy(context.Background(), admission, store, cfg, func() bool { return true }, logging.Testing()) + + require.Zero(t, admission.callCount(), "same-revision configuration conflicts must remain rejected") + }) + + t.Run("preserves committed capability activations on a revision bump", func(t *testing.T) { + t.Parallel() + + store := newReconcilerTestStore(t) + cfg := validBaseConfig() + batch := store.OpenWriteSession() + require.NoError(t, state.SaveClusterPolicy(batch, &commonpb.ClusterPolicy{ + Revision: cfg.ClusterPolicyRevision - 1, + CapabilityActivations: []*commonpb.CapabilityActivation{{ + Id: 7, Active: true, + }}, + })) + require.NoError(t, batch.Commit()) + + admission := &fakeAdmission{} + reconcileClusterPolicy(context.Background(), admission, store, cfg, func() bool { return true }, logging.Testing()) + + require.Equal(t, 1, admission.callCount()) + require.Equal(t, []*commonpb.CapabilityActivation{{Id: 7, Active: true}}, admission.lastPolicy.GetCapabilityActivations()) + }) } diff --git a/internal/bootstrap/controller_routed_checkpoint_fence_test.go b/internal/bootstrap/controller_routed_checkpoint_fence_test.go index 34bb0d2e43..f6a91653b3 100644 --- a/internal/bootstrap/controller_routed_checkpoint_fence_test.go +++ b/internal/bootstrap/controller_routed_checkpoint_fence_test.go @@ -274,7 +274,7 @@ func startCheckpointFenceNode(t *testing.T, ctx context.Context, store *dal.Stor require.NoError(t, err) registry := state.NewStateRegistry(cacheStore, attributes.New()) shared := state.NewSharedState() - machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, nil, shared, signal.NewNotifications(), nil, "checkpoint-fence", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, nil, shared, signal.NewNotifications(), nil, "checkpoint-fence", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) t.Cleanup(machine.Close) recovery := state.NewRecovery(machine, store) @@ -296,7 +296,7 @@ func startCheckpointFenceNode(t *testing.T, ctx context.Context, store *dal.Stor serviceAddr = raftAddr } instanceID := []byte(fmt.Sprintf("%016d", id)) - member, err := membership.NewMembership(membership.NewPeerStore(store), raftTransport, pool, id, raftAddr, serviceAddr, instanceID, logger) + member, err := membership.NewMembership(membership.NewPeerStore(store), raftTransport, pool, id, raftAddr, serviceAddr, instanceID, logger, membership.NoopClusterPolicyProvider) require.NoError(t, err) responses := node.NewLocalResponses() applier, err := node.NewApplier(machine, recovery, synchronizer, sp, store, w, logger, meter, 1000, 1000, nil, member, responses) diff --git a/internal/bootstrap/join_preflight_test.go b/internal/bootstrap/join_preflight_test.go index 5bc556dcaa..f647cdd445 100644 --- a/internal/bootstrap/join_preflight_test.go +++ b/internal/bootstrap/join_preflight_test.go @@ -15,6 +15,7 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" + raftmembership "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/node" "github.com/formancehq/ledger/v3/internal/storage/wal" ) @@ -68,6 +69,56 @@ func TestShouldRunJoinPreflight(t *testing.T) { }) } +func TestRecoveredRefreshPeers(t *testing.T) { + peers := recoveredRefreshPeers(1, map[uint64]raftmembership.ConfChangeContext{ + 1: {RaftAddress: "self:7777"}, + 2: {RaftAddress: "peer:7777"}, + 3: {}, + }) + require.Equal(t, []node.Peer{{ID: 2, Address: "peer:7777"}}, peers) +} + +func TestShouldRunMembershipRefresh(t *testing.T) { + t.Parallel() + logger := logging.Testing() + + t.Run("fresh WAL skips refresh", func(t *testing.T) { + t.Parallel() + cfg := Config{RaftConfig: node.NodeConfig{ + Peers: []node.Peer{{ID: 1, Address: "leader:7777"}}, + WalDir: t.TempDir(), + }} + assert.False(t, shouldRunMembershipRefresh(cfg, logger)) + }) + + t.Run("joined WAL refreshes", func(t *testing.T) { + t.Parallel() + walDir := t.TempDir() + require.NoError(t, wal.MarkClusterJoined(walDir)) + cfg := Config{RaftConfig: node.NodeConfig{ + Peers: []node.Peer{{ID: 1, Address: "leader:7777"}}, + WalDir: walDir, + }} + assert.True(t, shouldRunMembershipRefresh(cfg, logger)) + }) + + t.Run("joined WAL without startup flags refreshes", func(t *testing.T) { + t.Parallel() + walDir := t.TempDir() + require.NoError(t, wal.MarkClusterJoined(walDir)) + cfg := Config{RaftConfig: node.NodeConfig{WalDir: walDir}} + assert.True(t, shouldRunMembershipRefresh(cfg, logger)) + }) + + t.Run("former bootstrap node refreshes", func(t *testing.T) { + t.Parallel() + walDir := t.TempDir() + require.NoError(t, wal.MarkClusterJoined(walDir)) + cfg := Config{RaftConfig: node.NodeConfig{Bootstrap: true, WalDir: walDir}} + assert.True(t, shouldRunMembershipRefresh(cfg, logger)) + }) +} + // TestJoinPreflightRunsBeforeRaftTraffic proves the EN-1436 ordering invariant // (flemzord review on #1478): the join preflight OnStart hook must run — and, // when it fails, abort startup — BEFORE any hook that lets inbound Raft traffic diff --git a/internal/bootstrap/join_transport_recovery_test.go b/internal/bootstrap/join_transport_recovery_test.go index fed29fd516..e5f38f274c 100644 --- a/internal/bootstrap/join_transport_recovery_test.go +++ b/internal/bootstrap/join_transport_recovery_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/stretchr/testify/require" + "go.etcd.io/raft/v3/raftpb" "go.opentelemetry.io/otel/metric/noop" "google.golang.org/grpc" @@ -136,12 +137,14 @@ func newJoinRecoveryNode(t *testing.T, id uint64, addr string, peers []node.Peer t.Cleanup(func() { require.NoError(t, pool.Close()) }) t.Cleanup(func() { require.NoError(t, servicePool.Close()) }) tr := node.NewTransport(logger, pool, meters, id, node.TransportConfig{Reception: []int{32, 32, 32}, Send: []int{32, 32, 32}}, "join-recovery", 1024, addr, addr) - member, err := membership.NewMembership(membership.NewPeerStore(store), tr, servicePool, id, addr, addr, instanceID, logger) + member, err := membership.NewMembership(membership.NewPeerStore(store), tr, servicePool, id, addr, addr, instanceID, logger, membership.NoopClusterPolicyProvider) require.NoError(t, err) cacheStore, err := cache.New(1000, meter) require.NoError(t, err) registry := state.NewStateRegistry(cacheStore, attributes.New()) - machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, nil, state.NewSharedState(), signal.NewNotifications(), nil, "join-recovery", 0, member.WriteConfChange) + machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, nil, state.NewSharedState(), signal.NewNotifications(), nil, "join-recovery", 0, func(entry *raftpb.Entry, session *dal.WriteSession) error { + return member.WriteConfChange(entry, session) + }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) t.Cleanup(machine.Close) recovery := state.NewRecovery(machine, store) diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go index 6b7b68845b..35042241d0 100644 --- a/internal/bootstrap/module.go +++ b/internal/bootstrap/module.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "go.etcd.io/raft/v3/raftpb" "go.opentelemetry.io/otel/metric" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.uber.org/fx" @@ -47,6 +48,7 @@ import ( "github.com/formancehq/ledger/v3/internal/infra/attributes" "github.com/formancehq/ledger/v3/internal/infra/bloom" "github.com/formancehq/ledger/v3/internal/infra/cache" + "github.com/formancehq/ledger/v3/internal/infra/capabilities" clusterhealth "github.com/formancehq/ledger/v3/internal/infra/health" raftmembership "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/monitoring/diskusage" @@ -58,6 +60,7 @@ import ( "github.com/formancehq/ledger/v3/internal/infra/transport" "github.com/formancehq/ledger/v3/internal/pkg/commands" "github.com/formancehq/ledger/v3/internal/pkg/network" + "github.com/formancehq/ledger/v3/internal/pkg/reconciler" "github.com/formancehq/ledger/v3/internal/pkg/signal" "github.com/formancehq/ledger/v3/internal/pkg/version" "github.com/formancehq/ledger/v3/internal/pkg/worker" @@ -80,6 +83,10 @@ import ( // It is used to decide whether a joining node needs to register as a learner. type walFreshStart bool +func publishCapabilityGate(membership *raftmembership.Membership, policy *commonpb.ClusterPolicy) error { + return membership.SetCapabilityGate(capabilities.ActiveIDs(policy)) +} + // nodeProvideResult groups the outputs of the Node provider so we can also // expose the WAL-fresh-start indicator through the fx dependency graph. type nodeProvideResult struct { @@ -101,6 +108,30 @@ func provideWAL(cfg node.NodeConfig, logger logging.Logger, meterProvider metric return wal.New(cfg.WalDir, logger, meterProvider.Meter("wal")) } +// newClusterPolicyReconciler wires the leader-only policy maintenance into the +// generic periodic reconciler. Keeping this assembly in a named provider keeps +// the Fx graph readable while leaving the policy logic in dedicated helpers. +func newClusterPolicyReconciler(admission ctrl.Admission, store *dal.Store, builder *plan.Builder, cfg Config, raftNode *node.Node, logger logging.Logger) *reconciler.Reconciler { + return reconciler.New(func(ctx context.Context) { + if !raftNode.IsLeader() { + return + } + if policy, err := query.ReadClusterPolicy(store); err == nil { + if err := raftNode.SetCapabilityGate(capabilities.ActiveIDs(policy)); err != nil { + logger.WithFields(map[string]any{"error": err}).Errorf("Invalid capability gate in committed cluster policy") + + return + } + } + if err := proposeAuditKeyIfNeeded(ctx, raftNode, builder, store); err != nil { + logger.WithFields(map[string]any{"error": err}).Errorf("Audit key initialization failed; will retry") + + return + } + reconcileClusterPolicy(ctx, admission, store, cfg, raftNode.IsLeader, logger) + }, clusterPolicyReconcileInterval) +} + func Module() fx.Option { return fx.Options( transport.Module(), @@ -213,8 +244,14 @@ func Module() fx.Option { // — applier, node, background workers — observes the FSM at its // recovered state. NewMachine deliberately skips this step so it does // not perform any Pebble read itself. - func(machine *state.Machine, store *dal.Store) (*state.Recovery, error) { - recovery := state.NewRecovery(machine, store) + func(machine *state.Machine, store *dal.Store, membership *raftmembership.Membership) (*state.Recovery, error) { + recovery := state.NewRecovery(machine, store, state.WithStateRecovered(func(recovered *state.FSMState) error { + if err := capabilities.ValidateSupportedActive(recovered.ClusterPolicy, capabilities.SupportedIDs()); err != nil { + return err + } + + return publishCapabilityGate(membership, recovered.ClusterPolicy) + })) if err := recovery.RecoverState(); err != nil { return nil, fmt.Errorf("recovering FSM state at boot: %w", err) } @@ -233,14 +270,19 @@ func Module() fx.Option { // constructors. raftmembership.NewPeerStore, fx.Annotate(func( - store *raftmembership.PeerStore, + peerStore *raftmembership.PeerStore, defaultTransport *node.DefaultTransport, servicePool *transport.ConnectionPool, cfg node.NodeConfig, + durableStore *dal.Store, logger logging.Logger, ) (*raftmembership.Membership, error) { - return raftmembership.NewMembership(store, defaultTransport, servicePool, cfg.NodeID, cfg.AdvertiseAddr, cfg.ServiceAdvertiseAddr, cfg.InstanceID, logger) - }, fx.ParamTags(``, ``, `name:"service"`, ``, ``)), + policyProvider := func() (*commonpb.ClusterPolicy, error) { + return query.ReadClusterPolicy(durableStore) + } + + return raftmembership.NewMembership(peerStore, defaultTransport, servicePool, cfg.NodeID, cfg.AdvertiseAddr, cfg.ServiceAdvertiseAddr, cfg.InstanceID, logger, policyProvider) + }, fx.ParamTags(``, ``, `name:"service"`, ``, ``, ``)), // Provide events.Proposer from the Raft node (used by event emitter to replicate cursor). // Events go through Builder.Run, which already holds the IndexTracker // mutex around its proposer.Propose call. Wrapping the node in a @@ -287,6 +329,7 @@ func Module() fx.Option { registry := state.NewStateRegistry(c, attrs) snapshotter := state.NewCacheSnapshotter(logger, registry, bloomFilters) + var machine *state.Machine m, err := state.NewMachine( logger, registry, @@ -300,11 +343,24 @@ func Module() fx.Option { bloomFilters, "", // Audit key is loaded from replicated state during recovery. cfg.NumscriptCacheSize, - membership.WriteConfChange, + func(entry *raftpb.Entry, session *dal.WriteSession) error { + var policy *commonpb.ClusterPolicy + if machine != nil { + policy = machine.ClusterPolicy() + } + + return membership.WriteConfChange(entry, session, policy) + }, + func(policy *commonpb.ClusterPolicy) { + if err := publishCapabilityGate(membership, policy); err != nil { + logger.WithFields(map[string]any{"error": err}).Errorf("Publishing committed capability policy failed") + } + }, ) if err != nil { return nil, err } + machine = m logger.WithFields(map[string]any{ "duration": time.Since(machineStart).String(), @@ -326,6 +382,7 @@ func Module() fx.Option { Recovery *state.Recovery Synchronizer *state.Synchronizer Membership *raftmembership.Membership + Store *dal.Store LocalResponses node.LocalResponses }, ) (nodeProvideResult, error) { @@ -344,6 +401,23 @@ func Module() fx.Option { "snapshotTerm": snapshot.GetMetadata().GetTerm(), }).Infof("WAL fresh start detection") + var raftNode *node.Node + proposalAdmission := func(_ context.Context, proposal *node.Proposal) error { + policy, err := query.ReadClusterPolicy(params.Store) + if err != nil { + return fmt.Errorf("reading cluster capability policy before Raft proposal: %w", err) + } + if raftNode == nil { + return errors.New("raft proposal gate is not initialized") + } + // Take one immutable membership snapshot for both policy checks. The + // gate runs under the serialized proposal/index boundary; taking two + // snapshots here needlessly doubles lock, map-copy and allocation cost. + members := raftNode.CapabilityMembers() + + return validateProposalCapabilityAdmission(policy, proposal, members) + } + n, err := node.NewNode( params.NodeConfig, params.Transport, @@ -356,10 +430,12 @@ func Module() fx.Option { params.Synchronizer, params.Membership, params.LocalResponses, + node.WithProposalAdmission(proposalAdmission), ) if err != nil { return nodeProvideResult{}, err } + raftNode = n return nodeProvideResult{Node: n, FreshStart: freshStart}, nil }, @@ -574,6 +650,7 @@ func Module() fx.Option { store, logger, node, + node.MembershipCoordinator(), builder, meterProvider, hc, @@ -639,19 +716,7 @@ func Module() fx.Option { return clusterhealth.NewGRPCHealthUpdater(n, hs, clusterPolicyReady, rs.ReadProjectionHealthy) }, - func(admission ctrl.Admission, store *dal.Store, builder *plan.Builder, cfg Config, raftNode *node.Node, logger logging.Logger) *ClusterPolicyReconciler { - return NewClusterPolicyReconciler(func(ctx context.Context) { - if !raftNode.IsLeader() { - return - } - if err := proposeAuditKeyIfNeeded(ctx, raftNode, builder, store); err != nil { - logger.WithFields(map[string]any{"error": err}).Errorf("Audit key initialization failed; will retry") - - return - } - reconcileClusterPolicy(ctx, admission, store, cfg, raftNode.IsLeader, logger) - }) - }, + newClusterPolicyReconciler, ), fx.Decorate(func( params struct { @@ -879,9 +944,7 @@ func Module() fx.Option { })) }, // Wire Observer: handle LeadershipChange and LeaderReady events. - // ConfChange events are no longer dispatched here — Membership - // owns the transport / service-pool wiring and reacts directly - // to peer mutations (EN-1413). + // Membership owns transport / service-pool wiring. func( n *node.Node, store *dal.Store, @@ -892,6 +955,7 @@ func Module() fx.Option { mirrorManager *mirror.Manager, backupOrchestrator *backupapp.Orchestrator, healthChecker *clusterhealth.HealthChecker, + reconciler *reconciler.Reconciler, ) { n.SetObserver(node.NewObserver(func(event any) { switch e := event.(type) { @@ -928,9 +992,19 @@ func Module() fx.Option { } })) }, - func(lc fx.Lifecycle, node *node.Node, defaultTransport *node.DefaultTransport, logger logging.Logger) (*node.Node, error) { + func(lc fx.Lifecycle, node *node.Node, defaultTransport *node.DefaultTransport, membership *raftmembership.Membership, cfg Config, logger logging.Logger) (*node.Node, error) { lc.Append(nodeHook(node, logger)) + // A WAL that already joined the cluster must not take the learner + // bootstrap path again. It does, however, need to publish the + // capabilities of the restarted binary and any changed endpoints. + // This hook runs after the Raft server, transport and node are live; + // an UpdateNode for an existing voter may need that node to contribute + // to the commit quorum. + if shouldRunMembershipRefresh(cfg, logger) { + lc.Append(membershipRefreshHook(cfg, membership, logger)) + } + return node, nil }, func(lc fx.Lifecycle, observer *logintegrity.Observer) { @@ -997,7 +1071,7 @@ func Module() fx.Option { func(lc fx.Lifecycle, manager *mirror.Manager) { lc.Append(worker.FxHook(manager)) }, - func(lc fx.Lifecycle, reconciler *ClusterPolicyReconciler) { + func(lc fx.Lifecycle, reconciler *reconciler.Reconciler) { lc.Append(worker.FxHook(reconciler)) }, func(lc fx.Lifecycle, cfg Config, logger logging.Logger, raftNode *node.Node, store *dal.Store, machine *state.Machine, builder *plan.Builder) { @@ -1102,6 +1176,35 @@ func Module() fx.Option { ) } +// validateProposalCapabilityAdmission is the production proposal gate shared +// by every Node.Propose caller. The proposal carries typed declarations derived +// from the final command, while policy and membership are the committed +// snapshots captured at the serialized admission boundary. +func validateProposalCapabilityAdmission(policy *commonpb.ClusterPolicy, proposal *node.Proposal, members []capabilities.Member) error { + if proposal == nil { + return errors.New("raft proposal gate received a nil proposal") + } + + if err := capabilities.ValidateActive( + capabilities.SupportedIDs(), + proposal.ActivePolicyCapabilities(), + members, + ); err != nil { + return err + } + + state := capabilities.FromPolicy(policy) + if err := state.Validate(); err != nil { + return err + } + decision := capabilities.Evaluate(state, proposal.RequiredCapabilities(), false, members) + if !decision.Allowed { + return fmt.Errorf("%w: %s", capabilities.ErrCapabilityUnsupported, decision.Reason) + } + + return nil +} + // JoinAuthError is returned when a joining node is rejected by the target // cluster's inter-node RaftServer with codes.Unauthenticated. This is always // a cluster-secret misconfiguration (EN-1080): the join flow carries no user @@ -1199,6 +1302,19 @@ func shouldRunJoinPreflight(cfg Config, logger logging.Logger) bool { return true } +// shouldRunMembershipRefresh selects the normal-restart path. A fresh WAL is +// handled by joinPreflightHook. Restart peers come from recovered membership, +// so neither --join nor --bootstrap is required on subsequent boots. +func shouldRunMembershipRefresh(cfg Config, logger logging.Logger) bool { + if !wal.IsClusterJoined(cfg.RaftConfig.WalDir) { + return false + } + + logger.Infof("CLUSTER_JOINED marker present, refreshing membership evidence") + + return true +} + func registerSnapshotServiceLifecycle(lc fx.Lifecycle, server snapshotpb.SnapshotServiceServer) { lc.Append(fx.Hook{ OnStart: func(_ context.Context) error { @@ -1280,6 +1396,120 @@ func joinPreflightHook(cfg Config, logger logging.Logger) fx.Hook { } } +// newMembershipPeerDialer builds the common outbound gRPC dialer used by the +// learner bootstrap and membership-refresh paths. Keeping credential and +// option construction in one place prevents those two startup paths from +// drifting while still allowing each call site to control connection lifetime. +func newMembershipPeerDialer(cfg Config, tlsCfg TLSConfig) (func(string) (*ggrpc.ClientConn, error), error) { + creds, _, err := ClientTransportCredentials(tlsCfg) + if err != nil { + return nil, err + } + + dialOpts := []ggrpc.DialOption{ggrpc.WithTransportCredentials(creds)} + if cfg.ClusterSecret != "" { + dialOpts = append(dialOpts, transport.BearerTokenDialOption(cfg.ClusterSecret)) + } + + return func(address string) (*ggrpc.ClientConn, error) { + return ggrpc.NewClient(address, dialOpts...) + }, nil +} + +func membershipRefreshHook(cfg Config, membership *raftmembership.Membership, logger logging.Logger) fx.Hook { + var cancel context.CancelFunc + + return fx.Hook{ + OnStart: func(ctx context.Context) error { + cfg.RaftConfig.Peers = recoveredRefreshPeers(cfg.RaftConfig.NodeID, membership.PeerAddresses()) + if len(cfg.RaftConfig.Peers) == 0 { + return nil // A single-member cluster has no remote membership to refresh. + } + logger.WithFields(map[string]any{ + "nodeID": cfg.RaftConfig.NodeID, + "raftAddress": cfg.RaftConfig.AdvertiseAddr, + "serviceAddress": cfg.ServiceAdvertiseAddr(), + }).Infof("Restart mode: refreshing membership capabilities") + + // Perform one bounded synchronous attempt before the node starts serving. A + // permanent rejection (for example a removed incarnation or invalid + // identity) must abort startup rather than letting stale WAL membership + // evidence escape into the cluster. Transient connectivity failures + // (including Unavailable) and rolling-upgrade compatibility responses + // are handed to the background retry loop; permanent rejections remain + // startup failures. + err := tryRefreshMembershipForStartup(ctx, cfg, cfg.TLSConfig, logger) + if err == nil { + return nil + } + if !errors.Is(err, errMembershipRefreshTransient) { + return err + } + + refreshCtx, refreshCancel := context.WithCancel(context.Background()) + cancel = refreshCancel + go func() { + if err := tryRefreshMembership(refreshCtx, cfg, cfg.TLSConfig, logger); err != nil { + // TODO(rc.3): remove this compatibility fallback once every + // supported peer implements RefreshMembership. The leader-side + // reconciler keeps retrying transient evidence until convergence. + logger.WithFields(map[string]any{"error": err}).Errorf("Membership capability refresh failed; continuing with conservative evidence") + } + }() + + return nil + }, + OnStop: func(context.Context) error { + if cancel != nil { + cancel() + } + + return nil + }, + } +} + +func recoveredRefreshPeers(selfID uint64, addresses map[uint64]raftmembership.ConfChangeContext) []node.Peer { + peers := make([]node.Peer, 0, len(addresses)) + for id, address := range addresses { + if id != selfID && address.RaftAddress != "" { + peers = append(peers, node.Peer{ID: id, Address: address.RaftAddress}) + } + } + + return peers +} + +const membershipRefreshStartupTimeout = 5 * time.Second + +// tryRefreshMembershipForStartup gives the synchronous startup attempt a +// bounded context. A leader can accept a RefreshMembership RPC while a Raft +// proposal is waiting for a quorum; that proposal must not hold the lifecycle +// start hook until the process-wide startup deadline. Once this attempt times +// out, the caller can continue with conservative evidence and the lifecycle- +// owned retry loop can converge when quorum returns. +func tryRefreshMembershipForStartup(ctx context.Context, cfg Config, tlsCfg TLSConfig, logger logging.Logger) error { + return tryRefreshMembershipForStartupWithTimeout(ctx, cfg, tlsCfg, logger, membershipRefreshStartupTimeout) +} + +func tryRefreshMembershipForStartupWithTimeout(ctx context.Context, cfg Config, tlsCfg TLSConfig, logger logging.Logger, timeout time.Duration) error { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + err := tryRefreshMembershipOnce(attemptCtx, cfg, tlsCfg, logger) + if err == nil { + return nil + } + + // Preserve lifecycle cancellation as a hard error. The bounded attempt's + // expiry and a remote RPC deadline are transient while the lifecycle is live. + if ctx.Err() == nil && (errors.Is(attemptCtx.Err(), context.DeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded) { + return fmt.Errorf("%w: synchronous refresh attempt exceeded %s: %w", errMembershipRefreshTransient, timeout, err) + } + + return err +} + // tryAddLearner attempts to register this node as a learner on an existing // cluster, using the inter-node ClusterBootstrapService exposed on the // RaftServer. It retries on transient errors (Unavailable, no leader) @@ -1307,18 +1537,14 @@ func tryAddLearner(ctx context.Context, cfg Config, tlsCfg TLSConfig, logger log RaftAddress: cfg.RaftConfig.AdvertiseAddr, ServiceAddress: cfg.ServiceAdvertiseAddr(), InstanceId: instanceID, + Capabilities: capabilities.IDsAsUint32(capabilities.SupportedIDs()), } - creds, _, err := ClientTransportCredentials(tlsCfg) + dialPeer, err := newMembershipPeerDialer(cfg, tlsCfg) if err != nil { return fmt.Errorf("loading TLS credentials for learner registration: %w", err) } - dialOpts := []ggrpc.DialOption{ggrpc.WithTransportCredentials(creds)} - if cfg.ClusterSecret != "" { - dialOpts = append(dialOpts, transport.BearerTokenDialOption(cfg.ClusterSecret)) - } - backoff := 500 * time.Millisecond maxBackoff := 5 * time.Second @@ -1329,7 +1555,7 @@ func tryAddLearner(ctx context.Context, cfg Config, tlsCfg TLSConfig, logger log for { // Try each peer in order until one succeeds. for _, peer := range peers { - conn, err := ggrpc.NewClient(peer.Address, dialOpts...) + conn, err := dialPeer(peer.Address) if err != nil { logger.WithFields(map[string]any{ "peer": peer.ID, @@ -1462,6 +1688,106 @@ func tryAddLearner(ctx context.Context, cfg Config, tlsCfg TLSConfig, logger log } } +// tryRefreshMembership publishes this incarnation's current endpoints and +// capabilities through the leader. It is deliberately separate from +// tryAddLearner: a node with a populated WAL already owns Raft progress and +// must never be admitted through the empty-WAL learner path. +func tryRefreshMembership(ctx context.Context, cfg Config, tlsCfg TLSConfig, logger logging.Logger) error { + backoff := 500 * time.Millisecond + for { + err := tryRefreshMembershipOnce(ctx, cfg, tlsCfg, logger) + if err == nil || !errors.Is(err, errMembershipRefreshTransient) { + return err + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(backoff): + } + backoff = min(backoff*2, 5*time.Second) + } +} + +var errMembershipRefreshTransient = errors.New("membership refresh temporarily unavailable") + +func tryRefreshMembershipOnce(ctx context.Context, cfg Config, tlsCfg TLSConfig, logger logging.Logger) error { + instanceID, err := wal.EnsureInstanceID(cfg.RaftConfig.WalDir) + if err != nil { + return fmt.Errorf("ensuring instance id for membership refresh: %w", err) + } + + req := &clusterbootstrappb.RefreshMembershipRequest{ + NodeId: cfg.RaftConfig.NodeID, + RaftAddress: cfg.RaftConfig.AdvertiseAddr, + ServiceAddress: cfg.ServiceAdvertiseAddr(), + InstanceId: instanceID, + Capabilities: capabilities.IDsAsUint32(capabilities.SupportedIDs()), + } + + dialPeer, err := newMembershipPeerDialer(cfg, tlsCfg) + if err != nil { + return fmt.Errorf("loading TLS credentials for membership refresh: %w", err) + } + if cfg.ClusterID != "" { + ctx = metadata.AppendToOutgoingContext(ctx, node.MetadataKeyClusterID, cfg.ClusterID) + } + + var lastTransient error + for _, peer := range cfg.RaftConfig.Peers { + conn, dialErr := dialPeer(peer.Address) + if dialErr != nil { + logger.WithFields(map[string]any{"peer": peer.ID, "address": peer.Address, "error": dialErr}).Infof("Failed to dial peer for membership refresh") + lastTransient = dialErr + + continue + } + + _, callErr := clusterbootstrappb.NewClusterBootstrapServiceClient(conn).RefreshMembership(ctx, req) + _ = conn.Close() + if callErr == nil { + logger.WithFields(map[string]any{"peer": peer.ID}).Infof("Successfully refreshed membership capabilities") + + return nil + } + + if ctx.Err() != nil { + return ctx.Err() + } + + if !isRetryableMembershipRefreshError(callErr) { + return fmt.Errorf("failed to refresh membership via peer %d (%s): %w", peer.ID, peer.Address, callErr) + } + lastTransient = callErr + } + if err := ctx.Err(); err != nil { + return err + } + if lastTransient != nil { + return fmt.Errorf("%w: %w", errMembershipRefreshTransient, lastTransient) + } + + return errMembershipRefreshTransient +} + +func isRetryableMembershipRefreshError(err error) bool { + st, ok := status.FromError(err) + if !ok { + return false + } + + if st.Code() == codes.Unavailable || st.Code() == codes.DeadlineExceeded { + return true + } + + // TODO(rc.3): remove this compatibility retry once every supported peer + // implements RefreshMembership. During the rc.1/rc.2 rolling upgrade, an + // older peer returns Unimplemented rather than Unavailable for this newly + // introduced RPC; treating it as transient lets the node keep trying newer + // peers without making the startup hook fail immediately. + return st.Code() == codes.Unimplemented +} + // isStaleRaftProgress reports whether a FailedPrecondition status carries the // EN-1436 STALE_RAFT_PROGRESS ErrorInfo reason, distinguishing a stale-raft- // progress join rejection (remediation: `remove-node --force`) from the @@ -1576,13 +1902,18 @@ func proposeClusterConfigIfNeeded(parent context.Context, n plan.Proposer, build } } +// clusterPolicyReconcileInterval is how often the leader re-checks the applied +// cluster policy against the desired one. Short so a fresh leader commits the +// policy — and thereby opens write readiness — promptly. +const clusterPolicyReconcileInterval = time.Second + // reconcileClusterPolicy drives the replicated cluster policy toward the desired // revision. It gates on leadership itself and is safe to call repeatedly: the -// ClusterPolicyReconciler invokes it on a ticker so a transient proposal failure +// generic reconciler invokes it on a ticker so a transient proposal failure // self-heals on the next tick. Unlike the cluster config, the policy flows -// through AdmitClusterPolicy as an operational SetClusterPolicyProposal. -// The write-readiness gate exempts this non-business proposal so it is never -// blocked by its own cluster-policy gate. +// through AdmitClusterPolicy as an internal audited SetClusterPolicy order so +// the checker can re-derive it; the write-readiness gate exempts this +// non-business proposal so it is never blocked by its own cluster-policy gate. func reconcileClusterPolicy(ctx context.Context, admission ctrl.Admission, store *dal.Store, cfg Config, isLeader func() bool, logger logging.Logger) { if !isLeader() { return @@ -1605,8 +1936,31 @@ func reconcileClusterPolicy(ctx context.Context, admission ctrl.Admission, store MetadataMaxEntityBytes: cfg.MetadataMaxEntityBytes, MetadataMaxCommandBytes: cfg.MetadataMaxCommandBytes, } + if applied != nil { + desired.CapabilityActivations = make([]*commonpb.CapabilityActivation, 0, len(applied.GetCapabilityActivations())) + for _, activation := range applied.GetCapabilityActivations() { + if activation != nil { + desired.CapabilityActivations = append(desired.CapabilityActivations, activation.CloneVT()) + } + } + } + // The operator-controlled revision is an independent monotonic guard. A + // capability-only reconciliation may advance a policy revision, but it must + // never make a stale leader's ordinary configuration look newer than the + // revision it was configured to own. A higher applied revision is therefore + // acceptable only when the operator-controlled fields still match; this lets + // later capability-only reconciliations continue without requiring another + // flag bump. appliedRev := applied.GetRevision() + if applied != nil && appliedRev > cfg.ClusterPolicyRevision && !sameClusterPolicyConfiguration(applied, desired) { + return + } + if applied != nil && appliedRev == cfg.ClusterPolicyRevision && !sameClusterPolicyConfiguration(applied, desired) { + logger.Errorf("cluster policy revision %d conflicts with the configured policy; refusing to overwrite it", appliedRev) + + return + } switch { case appliedRev > desired.GetRevision(): @@ -1635,6 +1989,25 @@ func reconcileClusterPolicy(ctx context.Context, admission ctrl.Admission, store } } +// sameClusterPolicyConfiguration compares the operator-controlled fields while +// deliberately ignoring capability activations. Capabilities are reconciled +// from membership evidence and may legitimately require a revision bump; all +// other fields must remain protected by the configured revision. +func sameClusterPolicyConfiguration(a, b *commonpb.ClusterPolicy) bool { + if a == nil || b == nil { + return a == b + } + + left := a.CloneVT() + right := b.CloneVT() + left.Revision = 0 + right.Revision = 0 + left.CapabilityActivations = nil + right.CapabilityActivations = nil + + return left.EqualVT(right) +} + // buildResponseSigner returns the Ed25519 response signer for the gRPC // service plane. // diff --git a/internal/bootstrap/module_refresh_membership_test.go b/internal/bootstrap/module_refresh_membership_test.go new file mode 100644 index 0000000000..5b596720bb --- /dev/null +++ b/internal/bootstrap/module_refresh_membership_test.go @@ -0,0 +1,174 @@ +package bootstrap + +import ( + "context" + "errors" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + logging "github.com/formancehq/go-libs/v5/pkg/observe/log" + + "github.com/formancehq/ledger/v3/internal/infra/node" + "github.com/formancehq/ledger/v3/internal/proto/clusterbootstrappb" +) + +func TestIsRetryableMembershipRefreshError(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + code codes.Code + want bool + }{ + {name: "unavailable", code: codes.Unavailable, want: true}, + {name: "remote deadline", code: codes.DeadlineExceeded, want: true}, + {name: "old peer missing rpc", code: codes.Unimplemented, want: true}, + {name: "invalid request", code: codes.InvalidArgument, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.want, isRetryableMembershipRefreshError(status.Error(tc.code, "refresh failed"))) + }) + } +} + +type retryingRefreshServer struct { + clusterbootstrappb.UnimplementedClusterBootstrapServiceServer + + attempts atomic.Int32 + firstErr error + failures int32 +} + +type blockingRefreshServer struct { + clusterbootstrappb.UnimplementedClusterBootstrapServiceServer +} + +func (blockingRefreshServer) RefreshMembership(ctx context.Context, _ *clusterbootstrappb.RefreshMembershipRequest) (*clusterbootstrappb.RefreshMembershipResponse, error) { + <-ctx.Done() + + return nil, ctx.Err() +} + +func (s *retryingRefreshServer) RefreshMembership(context.Context, *clusterbootstrappb.RefreshMembershipRequest) (*clusterbootstrappb.RefreshMembershipResponse, error) { + if s.attempts.Add(1) <= max(1, s.failures) { + return nil, s.firstErr + } + + return &clusterbootstrappb.RefreshMembershipResponse{}, nil +} + +func serveRefreshMembership(t *testing.T, server clusterbootstrappb.ClusterBootstrapServiceServer) string { + t.Helper() + + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer() + clusterbootstrappb.RegisterClusterBootstrapServiceServer(srv, server) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + return lis.Addr().String() +} + +func refreshTestConfig(t *testing.T, peerAddress string) Config { + t.Helper() + + return Config{ + ClusterID: "test-cluster", + TLSConfig: TLSConfig{Mode: TLSModeDisabled}, + RaftConfig: node.NodeConfig{ + NodeID: 2, + WalDir: t.TempDir(), + AdvertiseAddr: "127.0.0.1:7777", + Peers: []node.Peer{{ID: 1, Address: peerAddress}}, + }, + } +} + +func TestTryRefreshMembershipRetriesUnimplementedUntilPeerSupportsRPC(t *testing.T) { + t.Parallel() + + peer := &retryingRefreshServer{firstErr: status.Error(codes.Unimplemented, "old peer")} + cfg := refreshTestConfig(t, serveRefreshMembership(t, peer)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + require.NoError(t, tryRefreshMembership(ctx, cfg, cfg.TLSConfig, logging.Testing())) + require.Equal(t, int32(2), peer.attempts.Load()) +} + +func TestTryRefreshMembershipRetriesRemoteDeadlinesUntilSuccess(t *testing.T) { + t.Parallel() + + peer := &retryingRefreshServer{firstErr: status.Error(codes.DeadlineExceeded, "leader proposal timed out"), failures: 2} + cfg := refreshTestConfig(t, serveRefreshMembership(t, peer)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + require.NoError(t, tryRefreshMembership(ctx, cfg, cfg.TLSConfig, logging.Testing())) + require.Equal(t, int32(3), peer.attempts.Load()) +} + +func TestTryRefreshMembershipStopsRetryingWhenContextIsCanceled(t *testing.T) { + t.Parallel() + + peer := &retryingRefreshServer{firstErr: status.Error(codes.Unavailable, "leader unavailable")} + cfg := refreshTestConfig(t, serveRefreshMembership(t, peer)) + + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 1) + go func() { result <- tryRefreshMembership(ctx, cfg, cfg.TLSConfig, logging.Testing()) }() + require.Eventually(t, func() bool { return peer.attempts.Load() > 0 }, time.Second, time.Millisecond) + cancel() + require.ErrorIs(t, <-result, context.Canceled) +} + +func TestTryRefreshMembershipPropagatesPermanentRejection(t *testing.T) { + t.Parallel() + + peer := &retryingRefreshServer{firstErr: status.Error(codes.FailedPrecondition, "removed incarnation")} + cfg := refreshTestConfig(t, serveRefreshMembership(t, peer)) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + t.Cleanup(cancel) + err := tryRefreshMembership(ctx, cfg, cfg.TLSConfig, logging.Testing()) + require.ErrorContains(t, err, "removed incarnation") + require.Equal(t, int32(1), peer.attempts.Load(), "permanent rejection must not be retried") +} + +func TestTryRefreshMembershipForStartupTreatsBoundedAttemptAsTransient(t *testing.T) { + t.Parallel() + + cfg := refreshTestConfig(t, serveRefreshMembership(t, blockingRefreshServer{})) + err := tryRefreshMembershipForStartupWithTimeout( + context.Background(), + cfg, + cfg.TLSConfig, + logging.Testing(), + 10*time.Millisecond, + ) + + require.ErrorIs(t, err, errMembershipRefreshTransient) + require.True(t, errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded) +} + +func TestTryRefreshMembershipForStartupTreatsEarlyRPCDeadlineAsTransient(t *testing.T) { + t.Parallel() + + peer := &retryingRefreshServer{firstErr: status.Error(codes.DeadlineExceeded, "leader proposal timed out")} + cfg := refreshTestConfig(t, serveRefreshMembership(t, peer)) + err := tryRefreshMembershipForStartupWithTimeout( + context.Background(), cfg, cfg.TLSConfig, logging.Testing(), time.Second, + ) + + require.ErrorIs(t, err, errMembershipRefreshTransient) + require.Equal(t, int32(1), peer.attempts.Load()) +} diff --git a/internal/bootstrap/node_hook_commit_drain_test.go b/internal/bootstrap/node_hook_commit_drain_test.go index 61401c1e91..50183f6a31 100644 --- a/internal/bootstrap/node_hook_commit_drain_test.go +++ b/internal/bootstrap/node_hook_commit_drain_test.go @@ -99,9 +99,9 @@ func TestNodeHookStopCancellationRetainsRealCommitDrain(t *testing.T) { command.ExecutionPlan = &raftcmdpb.ExecutionPlan{Attributes: []*raftcmdpb.AttributeCoverage{{ Id: &raftcmdpb.AttributeID{Id: ledgerKey[:]}, AttrCode: uint32(dal.SubAttrLedger), }}} - data, err := command.MarshalVT() + proposal, err := node.NewProposal(command.GetId(), command) require.NoError(t, err) - future, err := n.Propose(ctx, node.NewProposal(command.GetId(), data)) + future, err := n.Propose(ctx, proposal) require.NoError(t, err) select { case <-commitEntered: @@ -185,7 +185,7 @@ func newHookCommitDrainNode(t *testing.T, notifier state.Notifier) (*observedDra registry := state.NewStateRegistry(cacheStore, attributes.New()) machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, nil, state.NewSharedState(), notifier, nil, "hook-drain", 0, - func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) t.Cleanup(machine.Close) recovery := state.NewRecovery(machine, store) @@ -205,7 +205,7 @@ func newHookCommitDrainNode(t *testing.T, notifier state.Notifier) (*observedDra }) instanceID := []byte("0000000000000001") member, err := membership.NewMembership(membership.NewPeerStore(store), raftTransport, servicePool, - 1, "node-1:7000", "node-1:8000", instanceID, logger) + 1, "node-1:7000", "node-1:8000", instanceID, logger, membership.NoopClusterPolicyProvider) require.NoError(t, err) responses := node.NewLocalResponses() applier, err := node.NewApplier(machine, recovery, synchronizer, sp, store, w, logger, meter, 1000, 1000, nil, member, responses) diff --git a/internal/bootstrap/propose_cluster_config_defaults_test.go b/internal/bootstrap/propose_cluster_config_defaults_test.go index f73b1566d2..2df393dd0f 100644 --- a/internal/bootstrap/propose_cluster_config_defaults_test.go +++ b/internal/bootstrap/propose_cluster_config_defaults_test.go @@ -82,7 +82,7 @@ func TestProposeClusterConfig_DefaultThreshold(t *testing.T) { registry := state.NewStateRegistry(c, attrs) machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, nil, state.NewSharedState(), signal.NewNotifications(), nil, - "threshold-test", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + "threshold-test", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) t.Cleanup(machine.Close) result, err := machine.ApplyEntries(t.Context(), store, &raftpb.Entry{ diff --git a/internal/bootstrap/service_address_test.go b/internal/bootstrap/service_address_test.go index 0979151a86..b989bd30be 100644 --- a/internal/bootstrap/service_address_test.go +++ b/internal/bootstrap/service_address_test.go @@ -69,10 +69,10 @@ func TestServiceAdvertiseAddr_IPv6RemoteRPC(t *testing.T) { t.Cleanup(func() { require.NoError(t, pool.Close()) }) peerStore := membership.NewPeerStore(newTestStore(t)) remote, err := membership.NewMembership(peerStore, serviceAddressRaftTransport{}, pool, - 1, "", "", []byte("self-instance-id"), logging.Testing()) + 1, "", "", []byte("self-instance-id"), logging.Testing(), membership.NoopClusterPolicyProvider) require.NoError(t, err) require.NoError(t, remote.Register(published.NodeID, published.AdvertiseAddr, - published.ServiceAdvertiseAddr, published.InstanceID)) + published.ServiceAdvertiseAddr, published.InstanceID, nil)) remote.Start() conn := pool.GetConnection(published.NodeID) require.NotNil(t, conn) diff --git a/internal/domain/processing/processor_cluster_policy.go b/internal/domain/processing/processor_cluster_policy.go index 7bc4d7ad65..cbaeeb8220 100644 --- a/internal/domain/processing/processor_cluster_policy.go +++ b/internal/domain/processing/processor_cluster_policy.go @@ -1,6 +1,8 @@ package processing import ( + "fmt" + "github.com/formancehq/ledger/v3/internal/domain" "github.com/formancehq/ledger/v3/internal/proto/commonpb" "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" @@ -40,10 +42,39 @@ func applyClusterPolicy(newPolicy *commonpb.ClusterPolicy, scope Scope) (*common if err := validateClusterPolicyMetadataLimits(newPolicy); err != nil { return nil, err } + seenCapabilities := make(map[uint32]struct{}, len(newPolicy.GetCapabilityActivations())) + for _, activation := range newPolicy.GetCapabilityActivations() { + if activation == nil || activation.GetId() == 0 { + return nil, &domain.ErrClusterPolicyInvalid{Detail: "capability activation id must be non-zero"} + } + if _, exists := seenCapabilities[activation.GetId()]; exists { + return nil, &domain.ErrClusterPolicyInvalid{Detail: fmt.Sprintf("duplicate capability activation %d", activation.GetId())} + } + seenCapabilities[activation.GetId()] = struct{}{} + } current := scope.GetClusterPolicy() appliedRev := current.GetRevision() newRev := newPolicy.GetRevision() + active := make(map[uint32]struct{}) + for _, activation := range current.GetCapabilityActivations() { + if activation != nil && activation.GetActive() { + active[activation.GetId()] = struct{}{} + } + } + for id := range active { + found := false + for _, activation := range newPolicy.GetCapabilityActivations() { + if activation != nil && activation.GetId() == id && activation.GetActive() { + found = true + + break + } + } + if !found { + return nil, &domain.ErrClusterPolicyInvalid{Detail: fmt.Sprintf("capability %d cannot be deactivated", id)} + } + } switch { case newRev > appliedRev: diff --git a/internal/domain/processing/processor_cluster_policy_test.go b/internal/domain/processing/processor_cluster_policy_test.go index 02f974d02b..a83f8bafea 100644 --- a/internal/domain/processing/processor_cluster_policy_test.go +++ b/internal/domain/processing/processor_cluster_policy_test.go @@ -101,6 +101,29 @@ func TestProcessSetClusterPolicy_LowerRevisionStale(t *testing.T) { require.Equal(t, uint64(5), stale.AppliedRevision) } +func TestProcessSetClusterPolicy_CapabilityCannotDisappear(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mockStore := NewMockScope(ctrl) + processor, err := NewRequestProcessor(nil, 0) + require.NoError(t, err) + + current := withMetadataLimits(&commonpb.ClusterPolicy{ + Revision: 2, + CapabilityActivations: []*commonpb.CapabilityActivation{{Id: 7, Active: true}}, + QueryCheckpointLimit: 1, + }) + mockStore.EXPECT().GetClusterPolicy().Return(current) + + proposed := withMetadataLimits(&commonpb.ClusterPolicy{Revision: 3, QueryCheckpointLimit: 1}) + _, procErr := processor.ProcessOrder(clusterPolicyOrder(proposed), mockStore) + + var invalid *domain.ErrClusterPolicyInvalid + require.ErrorAs(t, procErr, &invalid) + require.Contains(t, invalid.Detail, "capability 7 cannot be deactivated") +} + // Structural validation runs before the revision check, so it needs no applied // policy: a missing policy, a zero revision, and a zero limit each reject. func TestProcessSetClusterPolicy_StructuralValidation(t *testing.T) { diff --git a/internal/infra/backup/audit_key_restore_test.go b/internal/infra/backup/audit_key_restore_test.go index 061e8a7a8f..611f47a64d 100644 --- a/internal/infra/backup/audit_key_restore_test.go +++ b/internal/infra/backup/audit_key_restore_test.go @@ -64,7 +64,7 @@ func recoveredAuditKeyMachine(t *testing.T, store *dal.Store) *state.Machine { registry := state.NewStateRegistry(c, attributes.New()) m, err := state.NewMachine(testLogger(), registry, state.NewCacheSnapshotter(testLogger(), registry, nil), store, dal.NewSentinelFactory(store, false), meter, keystore.NewKeyStore(), state.NewSharedState(), - signingParityNotifier{}, nil, "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + signingParityNotifier{}, nil, "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, state.NewRecovery(m, store).RecoverState()) diff --git a/internal/infra/backup/numscript_restore_parity_test.go b/internal/infra/backup/numscript_restore_parity_test.go index 3f33fef639..e1f9b94d8e 100644 --- a/internal/infra/backup/numscript_restore_parity_test.go +++ b/internal/infra/backup/numscript_restore_parity_test.go @@ -20,6 +20,7 @@ import ( "github.com/formancehq/ledger/v3/internal/infra/attributes" "github.com/formancehq/ledger/v3/internal/infra/cache" "github.com/formancehq/ledger/v3/internal/infra/health" + "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/node" "github.com/formancehq/ledger/v3/internal/infra/plan" "github.com/formancehq/ledger/v3/internal/infra/state" @@ -92,7 +93,7 @@ func newScriptedNode(t *testing.T, store *dal.Store) *scriptedNode { machine, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), store, dal.NewSentinelFactory(store, false), meters, keyStore, sharedState, signingParityNotifier{}, nil, - "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, state.NewRecovery(machine, store).RecoverState()) @@ -111,7 +112,7 @@ func newScriptedNode(t *testing.T, store *dal.Store) *scriptedNode { return &scriptedNode{ t: t, admission: admission.NewAdmission(store, logger, - &applyingProposer{t: t, machine: machine, store: store, tracker: tracker}, + &applyingProposer{t: t, machine: machine, store: store, tracker: tracker}, &membership.Coordinator{}, builder, meters, writeGate, keyStore, sharedState, attrs, numscript.NewNumscriptCache(0), func(context.Context) error { return nil }), } diff --git a/internal/infra/backup/signing_restore_parity_test.go b/internal/infra/backup/signing_restore_parity_test.go index 6c8e2400f8..c9cabd3a29 100644 --- a/internal/infra/backup/signing_restore_parity_test.go +++ b/internal/infra/backup/signing_restore_parity_test.go @@ -71,6 +71,7 @@ func newSigningParityMachine(t *testing.T) (*state.Machine, *dal.Store, *attribu signingParityAuditKey, 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, + state.NoopCapabilityPolicyCommitHandler, ) require.NoError(t, err) policy := &commonpb.ClusterPolicy{ diff --git a/internal/infra/capabilities/capabilities.go b/internal/infra/capabilities/capabilities.go new file mode 100644 index 0000000000..3e8f1064f8 --- /dev/null +++ b/internal/infra/capabilities/capabilities.go @@ -0,0 +1,324 @@ +// Package capabilities contains the deterministic contract used to gate +// rolling Raft protocol changes. It deliberately has no node-local state and +// no network or storage dependency: callers provide the committed policy and +// a snapshot of membership evidence. +package capabilities + +import ( + "bytes" + "errors" + "fmt" + "slices" + + "github.com/formancehq/ledger/v3/internal/proto/commonpb" + "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" +) + +// CapabilityID is the stable numeric identity of a consensus capability. +// The wire representation remains uint32 at protobuf and Raft JSON boundaries. +type CapabilityID uint32 + +// PreparedProposal is the opaque capability metadata produced while preparing +// a final proposal. Callers can inspect it through RequiredCapabilities, but +// cannot construct the declaration by passing an arbitrary slice to the node +// proposal factory. +type PreparedProposal struct { + required []CapabilityID +} + +// RequiredCapabilities returns a defensive copy of the derived declaration. +func (p PreparedProposal) RequiredCapabilities() []CapabilityID { + return slices.Clone(p.required) +} + +// SupportedIDs is the immutable capability set implemented by this binary. +// No protocol feature is activated yet; this empty registry keeps the +// membership-evidence plumbing ready for the first real capability. +func SupportedIDs() []CapabilityID { + return nil +} + +type MemberRole uint8 + +const ( + Voter MemberRole = iota + 1 + Learner +) + +type Member struct { + NodeID uint64 + InstanceID []byte + Role MemberRole + Reachable bool + Capabilities []CapabilityID +} + +type State struct { + Activations map[CapabilityID]bool +} + +type Blocker struct { + NodeID uint64 + Role MemberRole + Reason string +} + +type Decision struct { + Allowed bool + Reason string + Blockers []Blocker +} + +func ValidateMembers(members []Member) error { + seen := make(map[uint64][]byte, len(members)) + for _, member := range members { + if member.NodeID == 0 || (member.Role != Voter && member.Role != Learner) { + return fmt.Errorf("%w: invalid member %d", ErrInvalidState, member.NodeID) + } + identity := bytes.Clone(member.InstanceID) + if previous, ok := seen[member.NodeID]; ok && !bytes.Equal(previous, identity) { + return fmt.Errorf("%w: node %d has multiple incarnations", ErrInvalidState, member.NodeID) + } + if _, err := Normalize(member.Capabilities); err != nil { + return fmt.Errorf("%w: node %d has invalid capability evidence: %w", ErrInvalidState, member.NodeID, err) + } + seen[member.NodeID] = identity + } + + return nil +} + +var ( + ErrInvalidState = errors.New("invalid capability state") + ErrInvalidDeclaration = errors.New("invalid capability declaration") + ErrMissingCapability = errors.New("proposal is missing a required capability declaration") + ErrCapabilityUnsupported = errors.New("capability is unsupported by a cluster member") +) + +// RequiredCapabilities derives semantic protocol dependencies from the final +// proposal shape. The registry is deliberately colocated with the wire +// contract so a producer cannot make a feature-specific payload appear safe by +// omitting required_capabilities. Predicates inspect only replicated fields, +// which makes the result deterministic on every member. +func RequiredCapabilities(proposal *raftcmdpb.Proposal) ([]CapabilityID, error) { + if proposal == nil { + return nil, fmt.Errorf("%w: nil proposal", ErrInvalidDeclaration) + } + + return nil, nil +} + +// PrepareProposal derives and stamps the semantic capability declaration and +// returns the typed result for the caller to pass alongside the serialized +// proposal. Keeping the result avoids decoding the same proposal again during +// admission. +func PrepareProposal(proposal *raftcmdpb.Proposal) (PreparedProposal, error) { + required, err := RequiredCapabilities(proposal) + if err != nil { + return PreparedProposal{}, err + } + proposal.RequiredCapabilities = make([]uint32, len(required)) + for i, id := range required { + proposal.RequiredCapabilities[i] = uint32(id) + } + + return PreparedProposal{required: required}, nil +} + +// ValidateFinalProposal verifies that the declaration carried by the +// serialized proposal matches both its semantic payload and the metadata +// captured by the proposal factory. The factory runs this before queueing the +// proposal, keeping full decoding off the Raft orchestrator. +func ValidateFinalProposal(data []byte, declared []CapabilityID) error { + proposal := new(raftcmdpb.Proposal) + if err := proposal.UnmarshalVT(data); err != nil { + return fmt.Errorf("decode final proposal: %w", err) + } + expected, err := RequiredCapabilities(proposal) + if err != nil { + return err + } + wire := make([]CapabilityID, len(proposal.GetRequiredCapabilities())) + for i, id := range proposal.GetRequiredCapabilities() { + wire[i] = CapabilityID(id) + } + wire, err = Normalize(wire) + if err != nil { + return err + } + declared, err = Normalize(declared) + if err != nil { + return err + } + if !slices.Equal(wire, expected) { + return fmt.Errorf("%w: wire declaration does not match semantic payload", ErrMissingCapability) + } + if !slices.Equal(wire, declared) { + return fmt.Errorf("%w: declaration does not match final payload", ErrMissingCapability) + } + + return nil +} + +func (s State) Validate() error { + return nil +} + +func FromPolicy(policy *commonpb.ClusterPolicy) State { + s := State{Activations: make(map[CapabilityID]bool)} + if policy == nil { + return s + } + for _, activation := range policy.GetCapabilityActivations() { + if activation != nil { + s.Activations[CapabilityID(activation.GetId())] = activation.GetActive() + } + } + + return s +} + +// ActiveIDs returns the active capability IDs from a committed policy in +// deterministic order. Nil activations and inactive entries are ignored so +// callers can pass the result directly to membership admission gates. +func ActiveIDs(policy *commonpb.ClusterPolicy) []uint32 { + if policy == nil { + return nil + } + + active := make([]uint32, 0, len(policy.GetCapabilityActivations())) + for _, activation := range policy.GetCapabilityActivations() { + if activation != nil && activation.GetActive() { + active = append(active, activation.GetId()) + } + } + slices.Sort(active) + + return active +} + +// ValidateSupportedActive refuses a persisted policy that this binary cannot +// apply. It is used at boot and after installing a leader checkpoint. +func ValidateSupportedActive(policy *commonpb.ClusterPolicy, supported []CapabilityID) error { + for _, id := range ActiveIDs(policy) { + if !slices.Contains(supported, CapabilityID(id)) { + return fmt.Errorf("active capability %d is not supported by this binary", id) + } + } + + return nil +} + +// IDsAsUint32 converts the typed capability identifiers to their protobuf wire +// representation at an API boundary. +func IDsAsUint32(ids []CapabilityID) []uint32 { + result := make([]uint32, len(ids)) + for i, id := range ids { + result[i] = uint32(id) + } + + return result +} + +func Normalize(ids []CapabilityID) ([]CapabilityID, error) { + result := slices.Clone(ids) + slices.Sort(result) + for i := 1; i < len(result); i++ { + if result[i] == result[i-1] { + return nil, fmt.Errorf("%w: duplicate id %d", ErrInvalidDeclaration, result[i]) + } + } + + return result, nil +} + +// Ready returns the capabilities implemented by this binary and acknowledged +// by every current cluster member. Membership evidence is the acknowledgement +// recorded by JoinAsLearner/RefreshMembership; an empty member set is never +// considered ready. +func Ready(supported []CapabilityID, members []Member) ([]CapabilityID, error) { + if len(members) == 0 { + return nil, nil + } + supported, err := Normalize(supported) + if err != nil { + return nil, err + } + if err := ValidateMembers(members); err != nil { + return nil, err + } + ready := make([]CapabilityID, 0, len(supported)) + for _, id := range supported { + acknowledged := true + for _, member := range members { + if !slices.Contains(member.Capabilities, id) { + acknowledged = false + + break + } + } + if acknowledged { + ready = append(ready, id) + } + } + + return ready, nil +} + +// ValidateActive verifies that every active capability is acknowledged by +// every current member and is implemented by this binary. +func ValidateActive(supported, active []CapabilityID, members []Member) error { + if len(active) == 0 { + return nil + } + + ready, err := Ready(supported, members) + if err != nil { + return fmt.Errorf("validating capability activation evidence: %w", err) + } + for _, required := range active { + if !slices.Contains(ready, required) { + return fmt.Errorf("%w: capability %d is not acknowledged by every current member", ErrCapabilityUnsupported, required) + } + } + + return nil +} + +// Evaluate is pure. Reachability is reported separately from support: a +// temporarily unreachable voter is still a blocker for activation, while a +// learner is included because it must replay every committed entry. +func Evaluate(state State, required []CapabilityID, _ bool, members []Member) Decision { + if err := state.Validate(); err != nil { + return Decision{Reason: err.Error()} + } + required, err := Normalize(required) + if err != nil { + return Decision{Reason: err.Error()} + } + blockers := make([]Blocker, 0) + // A required capability is not admitted merely because members advertise + // support: the replicated activation must also have been committed. + for _, id := range required { + if !state.Activations[id] { + blockers = append(blockers, Blocker{Reason: fmt.Sprintf("capability %d is inactive", id)}) + } + } + for _, member := range members { + for _, id := range required { + if !slices.Contains(member.Capabilities, id) { + blockers = append(blockers, Blocker{NodeID: member.NodeID, Role: member.Role, Reason: fmt.Sprintf("missing capability %d", id)}) + + continue + } + if !member.Reachable { + blockers = append(blockers, Blocker{NodeID: member.NodeID, Role: member.Role, Reason: "member is unreachable"}) + } + } + } + if len(blockers) > 0 { + return Decision{Reason: ErrCapabilityUnsupported.Error(), Blockers: blockers} + } + + return Decision{Allowed: true, Blockers: blockers} +} diff --git a/internal/infra/capabilities/capabilities_bench_test.go b/internal/infra/capabilities/capabilities_bench_test.go new file mode 100644 index 0000000000..3d09e0dc69 --- /dev/null +++ b/internal/infra/capabilities/capabilities_bench_test.go @@ -0,0 +1,48 @@ +package capabilities + +import ( + "strconv" + "testing" +) + +// BenchmarkEvaluate records the deterministic cost shape of the proposal +// gate. The admission path supplies a lock-consistent membership snapshot and +// calls this pure function while holding its proposal/index serialization +// guards. Keep the matrix representative of a small production cluster and a +// larger deployment so changes to the O(members*required) work are visible. +func BenchmarkEvaluate(b *testing.B) { + for _, size := range []int{3, 7} { + for _, requiredCount := range []int{0, 1, 4} { + b.Run(memberCaseName(size, requiredCount), func(b *testing.B) { + state := State{Activations: make(map[CapabilityID]bool, requiredCount)} + required := make([]CapabilityID, requiredCount) + for i := range required { + id := CapabilityID(i + 1) + required[i] = id + state.Activations[id] = true + } + members := make([]Member, size) + for i := range members { + members[i] = Member{ + NodeID: uint64(i + 1), + Role: Voter, + Reachable: true, + Capabilities: append([]CapabilityID(nil), required...), + } + } + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if !Evaluate(state, required, false, members).Allowed { + b.Fatal("representative capability state was rejected") + } + } + }) + } + } +} + +func memberCaseName(memberCount, requiredCount int) string { + return "members=" + strconv.Itoa(memberCount) + "/required=" + strconv.Itoa(requiredCount) +} diff --git a/internal/infra/capabilities/capabilities_extra_test.go b/internal/infra/capabilities/capabilities_extra_test.go new file mode 100644 index 0000000000..b0c6df293d --- /dev/null +++ b/internal/infra/capabilities/capabilities_extra_test.go @@ -0,0 +1,12 @@ +package capabilities + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIncarnationValidation(t *testing.T) { + require.Error(t, ValidateMembers([]Member{{NodeID: 4, InstanceID: []byte("a"), Role: Voter}, {NodeID: 4, InstanceID: []byte("b"), Role: Learner}})) + require.NoError(t, ValidateMembers([]Member{{NodeID: 4, InstanceID: []byte("a"), Role: Voter}, {NodeID: 5, InstanceID: []byte("b"), Role: Learner}})) +} diff --git a/internal/infra/capabilities/capabilities_test.go b/internal/infra/capabilities/capabilities_test.go new file mode 100644 index 0000000000..15c581b4a7 --- /dev/null +++ b/internal/infra/capabilities/capabilities_test.go @@ -0,0 +1,147 @@ +package capabilities + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/formancehq/ledger/v3/internal/proto/commonpb" + "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" +) + +func capabilityState() State { + return State{Activations: map[CapabilityID]bool{}} +} + +func TestValidateSupportedActiveRefusesDowngrade(t *testing.T) { + policy := &commonpb.ClusterPolicy{CapabilityActivations: []*commonpb.CapabilityActivation{ + {Id: 7, Active: true}, + {Id: 8, Active: false}, + }} + require.NoError(t, ValidateSupportedActive(policy, []CapabilityID{7})) + require.ErrorContains(t, ValidateSupportedActive(policy, nil), "active capability 7") +} + +func TestEvaluateIncludesVotersAndLearners(t *testing.T) { + state := capabilityState() + state.Activations[99] = true + decision := Evaluate(state, []CapabilityID{99}, true, []Member{ + {NodeID: 1, Role: Voter, Reachable: true, Capabilities: []CapabilityID{99}}, + {NodeID: 2, Role: Learner, Reachable: true}, + }) + require.False(t, decision.Allowed) + require.Equal(t, uint64(2), decision.Blockers[0].NodeID) +} + +func TestEvaluateAllowsUnrelatedWriteWhenNoFeatureIsDeclared(t *testing.T) { + decision := Evaluate(capabilityState(), nil, true, []Member{{NodeID: 1, Role: Voter, Reachable: true}}) + require.True(t, decision.Allowed) +} + +func TestEvaluateRejectsInactiveCapabilityBeforeInspectingMembers(t *testing.T) { + decision := Evaluate(capabilityState(), []CapabilityID{7}, true, []Member{{ + NodeID: 1, + Role: Voter, + Reachable: true, + Capabilities: []CapabilityID{7}, + }}) + + require.False(t, decision.Allowed) + require.Contains(t, decision.Reason, "unsupported") + require.Empty(t, decision.Blockers[0].NodeID, "policy blockers do not identify a member") +} + +func TestEvaluateRejectsUnreachableMemberEvenWhenCapabilityIsAdvertised(t *testing.T) { + state := capabilityState() + state.Activations[7] = true + decision := Evaluate(state, []CapabilityID{7}, true, []Member{{ + NodeID: 1, + Role: Voter, + Reachable: false, + Capabilities: []CapabilityID{7}, + }}) + + require.False(t, decision.Allowed) + require.Len(t, decision.Blockers, 1) + require.Equal(t, uint64(1), decision.Blockers[0].NodeID) +} + +func TestEvaluateAllowsActiveCapabilityWhenAllMembersAdvertiseAndReachIt(t *testing.T) { + state := capabilityState() + state.Activations[7] = true + decision := Evaluate(state, []CapabilityID{7}, true, []Member{ + {NodeID: 1, Role: Voter, Reachable: true, Capabilities: []CapabilityID{7}}, + {NodeID: 2, Role: Learner, Reachable: true, Capabilities: []CapabilityID{7}}, + }) + + require.True(t, decision.Allowed) + require.Empty(t, decision.Blockers) +} + +func TestPrepareProposalClearsFutureDeclarations(t *testing.T) { + proposal := &raftcmdpb.Proposal{RequiredCapabilities: []uint32{99}} + prepared, err := PrepareProposal(proposal) + require.NoError(t, err) + require.Empty(t, prepared.RequiredCapabilities()) + require.Empty(t, proposal.GetRequiredCapabilities()) +} + +func TestFromPolicyRoundTrip(t *testing.T) { + state := FromPolicy(&commonpb.ClusterPolicy{ + CapabilityActivations: []*commonpb.CapabilityActivation{{Id: 99, Active: true}}, + }) + require.True(t, state.Activations[99]) + require.NoError(t, state.Validate()) +} + +func TestActiveIDsReturnsSortedActiveIDs(t *testing.T) { + t.Parallel() + + policy := &commonpb.ClusterPolicy{ + CapabilityActivations: []*commonpb.CapabilityActivation{ + {Id: 9, Active: true}, + nil, + {Id: 3, Active: false}, + {Id: 4, Active: true}, + }, + } + + require.Equal(t, []uint32{4, 9}, ActiveIDs(policy)) + require.Nil(t, ActiveIDs(nil)) +} + +func TestReadyReturnsIntersectionAcrossMembers(t *testing.T) { + members := []Member{ + {NodeID: 1, Role: Voter, Capabilities: []CapabilityID{1, 2}}, + {NodeID: 2, Role: Learner, Capabilities: []CapabilityID{1}}, + } + + ready, err := Ready([]CapabilityID{2, 1}, members) + require.NoError(t, err) + require.Equal(t, []CapabilityID{1}, ready) +} + +func TestReadyDoesNotActivateAnEmptyCluster(t *testing.T) { + ready, err := Ready([]CapabilityID{1}, nil) + require.NoError(t, err) + require.Empty(t, ready) +} + +func TestReadyRejectsMalformedMemberEvidence(t *testing.T) { + _, err := Ready([]CapabilityID{1}, []Member{{ + NodeID: 1, + Role: Voter, + Capabilities: []CapabilityID{1, 1}, + }}) + require.ErrorContains(t, err, "invalid capability evidence") +} + +func TestReadyIgnoresUnsupportedAdvertisedIDs(t *testing.T) { + ready, err := Ready([]CapabilityID{1}, []Member{{ + NodeID: 1, + Role: Voter, + Capabilities: []CapabilityID{1, 99}, + }}) + require.NoError(t, err) + require.Equal(t, []CapabilityID{1}, ready) +} diff --git a/internal/infra/membership/committed_peer_refresh_test.go b/internal/infra/membership/committed_peer_refresh_test.go index 846cfd7067..5eb92654f1 100644 --- a/internal/infra/membership/committed_peer_refresh_test.go +++ b/internal/infra/membership/committed_peer_refresh_test.go @@ -61,12 +61,13 @@ func TestCommittedPeerRefreshUpdatesRaftTransport(t *testing.T) { "self:8000", []byte("self-instance-id"), logger, + membership.NoopClusterPolicyProvider, ) require.NoError(t, err) m.Start() oldInstanceID := []byte("old-instance-id-") - require.NoError(t, m.Register(2, "old:7000", "old:8000", oldInstanceID)) + require.NoError(t, m.Register(2, "old:7000", "old:8000", oldInstanceID, nil)) require.Equal(t, "old:7000", raftPool.GetPeerAddress(2)) newInstanceID := []byte("new-instance-id-") @@ -88,7 +89,7 @@ func TestCommittedPeerRefreshUpdatesRaftTransport(t *testing.T) { // Node.finishReady performs this Set as soon as it observes the commit, // before submitting the entry to the asynchronous FSM applier. - require.NoError(t, m.Set(2, "new:7000", "new:8000", newInstanceID)) + require.NoError(t, m.Set(2, "new:7000", "new:8000", newInstanceID, nil)) require.Equal(t, "new:7000", m.PeerAddresses()[2].RaftAddress) require.Equal(t, newInstanceID, m.PeerAddresses()[2].InstanceID) diff --git a/internal/infra/membership/confchange.go b/internal/infra/membership/confchange.go index 805e9f8145..74e05482ed 100644 --- a/internal/infra/membership/confchange.go +++ b/internal/infra/membership/confchange.go @@ -3,7 +3,9 @@ package membership import ( "bytes" "encoding/json" + "errors" "fmt" + "slices" "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" @@ -23,6 +25,41 @@ func ValidateInstanceID(instanceID []byte) error { return nil } +// ValidateCapabilities enforces the canonical representation used in +// membership evidence. IDs are non-zero, sorted, and unique so the same +// concrete member cannot present two byte-level representations of support. +func ValidateCapabilities(ids []uint32) error { + for i, id := range ids { + if id == 0 { + return errors.New("capability id must be non-zero") + } + if i > 0 && ids[i-1] >= id { + return errors.New("capabilities must be sorted and unique") + } + } + + return nil +} + +// ValidateCapabilityEvidence validates the proof attached to a membership +// transition against all capabilities currently active in the committed +// policy. An empty policy keeps membership admission backwards-compatible. +func ValidateCapabilityEvidence(ctx *ConfChangeContext, required []uint32) error { + if ctx == nil { + return errors.New("missing capability membership evidence") + } + if err := ValidateCapabilities(ctx.Capabilities); err != nil { + return fmt.Errorf("invalid capability evidence: %w", err) + } + for _, id := range required { + if !slices.Contains(ctx.Capabilities, id) { + return fmt.Errorf("capability %d is required by the active policy", id) + } + } + + return nil +} + // ConfChangeContext carries peer addresses alongside a Raft ConfChange so that // all nodes (not just the leader) can learn the new peer's addresses when the // ConfChange is committed. @@ -35,6 +72,11 @@ type ConfChangeContext struct { RaftAddress string `json:"raftAddress"` ServiceAddress string `json:"serviceAddress"` InstanceID []byte `json:"instanceId,omitempty"` + // Capabilities are the signed-by-consensus membership evidence for this + // concrete peer incarnation. They travel in every registration context and + // are persisted in PeerAddress so activation cannot accidentally inherit + // support from a replaced pod. + Capabilities []uint32 `json:"capabilities,omitempty"` // ProposalID is a leader-local correlation token. It is committed with the // ConfChange so the proposing leader can resolve only the matching waiter; // FSM apply deliberately ignores it. @@ -48,7 +90,8 @@ type ConfChangeContext struct { func (c ConfChangeContext) Equal(other ConfChangeContext) bool { return c.RaftAddress == other.RaftAddress && c.ServiceAddress == other.ServiceAddress && - bytes.Equal(c.InstanceID, other.InstanceID) + bytes.Equal(c.InstanceID, other.InstanceID) && + slices.Equal(c.Capabilities, other.Capabilities) } // MarshalConfChangeContext serialises a ConfChangeContext to JSON bytes diff --git a/internal/infra/membership/confchange_identity_test.go b/internal/infra/membership/confchange_identity_test.go index 40ce1f37c9..5551348688 100644 --- a/internal/infra/membership/confchange_identity_test.go +++ b/internal/infra/membership/confchange_identity_test.go @@ -1,13 +1,118 @@ package membership import ( + "errors" "testing" "github.com/stretchr/testify/require" "go.etcd.io/raft/v3/raftpb" "google.golang.org/protobuf/proto" + + "github.com/formancehq/ledger/v3/internal/proto/commonpb" ) +func TestWriteConfChangeAllowsRegistrationBeforeInitialPolicy(t *testing.T) { + m := newTestMembership(t) + m.capabilityPolicyProvider = func() (*commonpb.ClusterPolicy, error) { + return nil, nil + } + + data, err := MarshalConfChangeContext(ConfChangeContext{ + RaftAddress: "peer:7777", + ServiceAddress: "peer:8888", + InstanceID: fixedInstanceID(1), + }) + require.NoError(t, err) + cc := &raftpb.ConfChangeV2{ + Changes: []*raftpb.ConfChangeSingle{{Type: new(raftpb.ConfChangeAddLearnerNode), NodeId: proto.Uint64(9)}}, + Context: data, + } + encoded, err := proto.Marshal(cc) + require.NoError(t, err) + + session := m.store.OpenWriteSession() + require.NoError(t, m.WriteConfChange(&raftpb.Entry{Type: new(raftpb.EntryConfChangeV2), Data: encoded}, session)) + require.NoError(t, session.Commit()) +} + +func TestWriteConfChangeFailsClosedOnPolicyReadError(t *testing.T) { + m := newTestMembership(t) + m.capabilityPolicyProvider = func() (*commonpb.ClusterPolicy, error) { + return nil, errors.New("pebble unavailable") + } + + data, err := MarshalConfChangeContext(ConfChangeContext{ + RaftAddress: "peer:7777", + ServiceAddress: "peer:8888", + InstanceID: fixedInstanceID(1), + }) + require.NoError(t, err) + cc := &raftpb.ConfChangeV2{ + Changes: []*raftpb.ConfChangeSingle{{Type: new(raftpb.ConfChangeAddLearnerNode), NodeId: proto.Uint64(9)}}, + Context: data, + } + encoded, err := proto.Marshal(cc) + require.NoError(t, err) + + session := m.store.OpenWriteSession() + require.ErrorContains(t, m.WriteConfChange(&raftpb.Entry{Type: new(raftpb.EntryConfChangeV2), Data: encoded}, session), "pebble unavailable") + require.NoError(t, session.Commit()) +} + +func TestValidateCapabilityEvidenceRequiresActiveCapabilities(t *testing.T) { + ctx := &ConfChangeContext{ + InstanceID: bytesN(0x42), + Capabilities: []uint32{1, 2}, + } + + require.NoError(t, ValidateCapabilityEvidence(ctx, []uint32{1, 2})) + require.ErrorContains(t, ValidateCapabilityEvidence(ctx, []uint32{1, 3}), "capability 3") + require.NoError(t, ValidateCapabilityEvidence(&ConfChangeContext{InstanceID: bytesN(0x42)}, nil)) +} + +func TestValidateTransitionEvidenceReadsCommittedPolicyBeforeCachedGate(t *testing.T) { + m := newTestMembership(t) + m.requiredCapabilities = nil // simulate the post-commit cache update window + m.capabilityPolicyProvider = func() (*commonpb.ClusterPolicy, error) { + return &commonpb.ClusterPolicy{ + CapabilityActivations: []*commonpb.CapabilityActivation{{Id: 7, Active: true}}, + }, nil + } + + err := m.ValidateTransitionEvidence(&ConfChangeContext{ + InstanceID: fixedInstanceID(1), + Capabilities: nil, + }) + require.ErrorContains(t, err, "capability 7 is required") +} + +func TestWriteConfChangeRejectsCapabilitiesMissingFromCommittedPolicy(t *testing.T) { + m := newTestMembership(t) + m.capabilityPolicyProvider = func() (*commonpb.ClusterPolicy, error) { + return &commonpb.ClusterPolicy{ + CapabilityActivations: []*commonpb.CapabilityActivation{{Id: 7, Active: true}}, + }, nil + } + + data, err := MarshalConfChangeContext(ConfChangeContext{ + RaftAddress: "peer:7777", + ServiceAddress: "peer:8888", + InstanceID: fixedInstanceID(1), + }) + require.NoError(t, err) + cc := &raftpb.ConfChangeV2{ + Changes: []*raftpb.ConfChangeSingle{{Type: new(raftpb.ConfChangeAddLearnerNode), NodeId: proto.Uint64(9)}}, + Context: data, + } + encoded, err := proto.Marshal(cc) + require.NoError(t, err) + + session := m.store.OpenWriteSession() + err = m.WriteConfChange(&raftpb.Entry{Type: new(raftpb.EntryConfChangeV2), Data: encoded}, session) + require.ErrorContains(t, err, "capability 7 is required") + require.NoError(t, session.Commit()) +} + func TestWriteConfChangeRejectsIncompleteRegistration(t *testing.T) { t.Parallel() for _, kind := range []raftpb.ConfChangeType{raftpb.ConfChangeAddNode, raftpb.ConfChangeAddLearnerNode, raftpb.ConfChangeUpdateNode} { diff --git a/internal/infra/membership/coordinator.go b/internal/infra/membership/coordinator.go new file mode 100644 index 0000000000..c4debc68fc --- /dev/null +++ b/internal/infra/membership/coordinator.go @@ -0,0 +1,69 @@ +package membership + +import ( + "context" + "sync" + "time" + + "github.com/formancehq/ledger/v3/internal/infra/state" + "github.com/formancehq/ledger/v3/internal/pkg/futures" +) + +// Coordinator serializes Raft membership changes with cluster policy updates. +// Its lock is held until a proposed policy has applied. +type Coordinator struct { + sync.Mutex + + waitForPriorMembership func(context.Context) error + stop <-chan struct{} + terminal <-chan struct{} +} + +// Configure connects the Raft barrier and lifecycle signals before the coordinator is used. +func (c *Coordinator) Configure(waitForPriorMembership func(context.Context) error, stop, terminal <-chan struct{}) { + c.waitForPriorMembership = waitForPriorMembership + c.stop = stop + c.terminal = terminal +} + +func (c *Coordinator) AcquireClusterPolicy(ctx context.Context) (func(*futures.Future[state.ApplyResult]), error) { + for !c.TryLock() { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + if c.waitForPriorMembership != nil { + if err := c.waitForPriorMembership(ctx); err != nil { + c.Unlock() + + return nil, err + } + } + + return c.releaseAfter, nil +} + +func (c *Coordinator) releaseAfter(future *futures.Future[state.ApplyResult]) { + if future == nil { + c.Unlock() + + return + } + go func() { + defer c.Unlock() + waitCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + select { + case <-c.stop: + cancel() + case <-c.terminal: + cancel() + case <-waitCtx.Done(): + } + }() + _, _ = future.Wait(waitCtx) + }() +} diff --git a/internal/infra/membership/identity_rejection_test.go b/internal/infra/membership/identity_rejection_test.go index 6a2726adcf..8b12879184 100644 --- a/internal/infra/membership/identity_rejection_test.go +++ b/internal/infra/membership/identity_rejection_test.go @@ -16,7 +16,7 @@ func TestWriteConfChangeRejectsRemovalWithoutContext(t *testing.T) { m := newTestMembership(t) identity := fixedInstanceID(9) - require.NoError(t, m.Register(9, "peer:7777", "peer:8888", identity)) + require.NoError(t, m.Register(9, "peer:7777", "peer:8888", identity, nil)) original := m.PeerAddresses() data, err := proto.Marshal(&raftpb.ConfChangeV2{Changes: []*raftpb.ConfChangeSingle{{ Type: new(raftpb.ConfChangeRemoveNode), NodeId: new(uint64(9)), diff --git a/internal/infra/membership/membership.go b/internal/infra/membership/membership.go index 7975279a5c..4c61368ec5 100644 --- a/internal/infra/membership/membership.go +++ b/internal/infra/membership/membership.go @@ -12,6 +12,8 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" + "github.com/formancehq/ledger/v3/internal/infra/capabilities" + "github.com/formancehq/ledger/v3/internal/proto/commonpb" "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" "github.com/formancehq/ledger/v3/internal/storage/dal" ) @@ -69,6 +71,12 @@ type Membership struct { mu sync.RWMutex addresses map[uint64]ConfChangeContext + // capabilityGate is updated from the committed ClusterPolicy projection. + // It is intentionally kept beside the membership cache so Add/Promote + // admission observes one lock-consistent view of active requirements and + // member identity evidence. + requiredCapabilities []uint32 + capabilityPolicyProvider func() (*commonpb.ClusterPolicy, error) // started gates transport / service-pool wiring. Until Start fires // (OnStart hook, after the local Raft gRPC server is listening), // Set / Remove / Register / Rehydrate mutate only the cache — the @@ -94,7 +102,15 @@ type Membership struct { // Failure to read the peer rows is fatal: without them the node would // boot with an empty cache while the WAL ConfState still claims the // cluster has voters. -func NewMembership(store *PeerStore, transport Transport, pool Pool, selfNodeID uint64, selfRaftAddr, selfServiceAddr string, selfInstanceID []byte, logger logging.Logger) (*Membership, error) { +// NoopClusterPolicyProvider is an explicit dependency for isolated membership +// users that do not have a durable ClusterPolicy projection (for example unit +// tests). Production bootstrap injects the Pebble-backed provider. +func NoopClusterPolicyProvider() (*commonpb.ClusterPolicy, error) { return nil, nil } + +func NewMembership(store *PeerStore, transport Transport, pool Pool, selfNodeID uint64, selfRaftAddr, selfServiceAddr string, selfInstanceID []byte, logger logging.Logger, policyProvider func() (*commonpb.ClusterPolicy, error)) (*Membership, error) { + if policyProvider == nil { + return nil, errors.New("cluster policy provider is required") + } if err := ValidateInstanceID(selfInstanceID); err != nil { return nil, fmt.Errorf("invalid self identity: %w", err) } @@ -109,15 +125,16 @@ func NewMembership(store *PeerStore, transport Transport, pool Pool, selfNodeID }).Infof("Loaded cluster membership from Pebble") return &Membership{ - store: store, - transport: transport, - pool: pool, - selfNodeID: selfNodeID, - selfRaftAddr: selfRaftAddr, - selfServiceAddr: selfServiceAddr, - selfInstanceID: bytes.Clone(selfInstanceID), - logger: logger, - addresses: addresses, + store: store, + transport: transport, + pool: pool, + selfNodeID: selfNodeID, + selfRaftAddr: selfRaftAddr, + selfServiceAddr: selfServiceAddr, + selfInstanceID: bytes.Clone(selfInstanceID), + logger: logger, + capabilityPolicyProvider: policyProvider, + addresses: addresses, }, nil } @@ -239,6 +256,61 @@ func (m *Membership) SnapshotDonorCandidates() []uint64 { return candidates } +// SetCapabilityGate publishes the committed activation gate used by +// membership transitions. The required IDs are copied and must be canonical. +func (m *Membership) SetCapabilityGate(required []uint32) error { + if err := ValidateCapabilities(required); err != nil { + return err + } + + m.mu.Lock() + defer m.mu.Unlock() + m.requiredCapabilities = append([]uint32(nil), required...) + + return nil +} + +// CapabilityGate returns the current committed activation gate. +func (m *Membership) CapabilityGate() (required []uint32) { + m.mu.RLock() + defer m.mu.RUnlock() + + return append([]uint32(nil), m.requiredCapabilities...) +} + +// ValidateTransitionEvidence checks a registration context against the latest +// locally published gate. The FSM write path revalidates the same evidence +// against the committed policy, which closes races with policy application. +func (m *Membership) ValidateTransitionEvidence(ctx *ConfChangeContext) error { + m.mu.RLock() + required := append([]uint32(nil), m.requiredCapabilities...) + provider := m.capabilityPolicyProvider + m.mu.RUnlock() + + // Read the durable policy at the admission boundary as well as the + // published cache. The cache is updated immediately after the Pebble + // commit; consulting the provider closes the small window where a newly + // activated capability is durable but the in-memory gate has not caught up + // yet. In that window, a proposal must be checked against the stricter, + // committed policy rather than admitted optimistically. + if provider != nil { + policy, err := provider() + if err != nil { + return fmt.Errorf("reading committed cluster policy: %w", err) + } + if policy != nil { + required = required[:0] + for _, activation := range policy.GetCapabilityActivations() { + if activation != nil && activation.GetActive() { + required = append(required, activation.GetId()) + } + } + } + } + + return ValidateCapabilityEvidence(ctx, required) +} + // GetInstanceID returns the peer's 16-byte identity UUID from the in-memory // cache, and ok=true when the peer is currently known. Returns (nil, false) // when the peer is not in the cache. Removal and promotion treat a missing @@ -269,10 +341,17 @@ func (m *Membership) GetInstanceID(nodeID uint64) ([]byte, bool) { // Cache mutation and transport wiring both happen inside the lock so // they stay in lockstep with a concurrent Rehydrate. See Rehydrate's // locking note. -func (m *Membership) Set(nodeID uint64, raftAddr, serviceAddr string, instanceID []byte) error { +// Set upserts a peer and persists its capability evidence in the +// in-memory registration. The FSM write path stores the same fields in the +// durable PeerAddress row, while finishReady uses this method to keep the +// transport cache and evidence synchronized before resolving the proposal. +func (m *Membership) Set(nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { if err := ValidateInstanceID(instanceID); err != nil { return fmt.Errorf("setting peer %d: %w", nodeID, err) } + if err := ValidateCapabilities(capabilityIDs); err != nil { + return fmt.Errorf("setting peer %d: %w", nodeID, err) + } m.mu.Lock() defer m.mu.Unlock() @@ -281,6 +360,7 @@ func (m *Membership) Set(nodeID uint64, raftAddr, serviceAddr string, instanceID RaftAddress: raftAddr, ServiceAddress: serviceAddr, InstanceID: bytes.Clone(instanceID), + Capabilities: append([]uint32(nil), capabilityIDs...), } m.wireAdd(nodeID, raftAddr, serviceAddr) @@ -305,15 +385,24 @@ func (m *Membership) Remove(nodeID uint64) { // Register writes a peer through Pebble (own session) AND the cache, in // lockstep. Used by lifecycle paths that bypass the FSM: bootstrap -// initial-peer persistence and ForceRemoveNode (dual: Unregister). -// instanceID is always the peer's 16-byte WAL/PVC identity. Identity-less -// membership rows are rejected rather than persisted as a partial member. -func (m *Membership) Register(nodeID uint64, raftAddr, serviceAddr string, instanceID []byte) error { - if err := m.store.Put(nodeID, raftAddr, serviceAddr, instanceID); err != nil { +// initial-peer persistence, self refresh, and ForceRemoveNode's dual +// Unregister operation. instanceID is always the peer's 16-byte WAL/PVC +// identity. Identity-less membership rows are rejected rather than persisted +// as a partial member. capabilityIDs carries the peer's capability evidence. +func (m *Membership) Register(nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { + // A restart refreshes the local addresses. Preserve evidence for another + // peer when no new proof is available, but never preserve self evidence: + // an empty self list is the binary's real list after a downgrade. + if len(capabilityIDs) == 0 && nodeID != m.selfNodeID { + if existing, ok := m.PeerAddresses()[nodeID]; ok && bytes.Equal(existing.InstanceID, instanceID) { + capabilityIDs = existing.Capabilities + } + } + if err := m.store.Put(nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs); err != nil { return fmt.Errorf("persisting peer %d: %w", nodeID, err) } - if err := m.Set(nodeID, raftAddr, serviceAddr, instanceID); err != nil { + if err := m.Set(nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs); err != nil { return err } @@ -527,6 +616,19 @@ func (m *Membership) Rehydrate() error { // checkpoint introduced an identity-less configured member would violate the // cluster's removal/admission contract. func (m *Membership) OnSnapshotInstalled() { + // The checkpoint and spool replay may have introduced a policy that this + // binary cannot execute. Stop before accepting any further Raft entries. + m.mu.RLock() + provider := m.capabilityPolicyProvider + m.mu.RUnlock() + policy, err := provider() + if err != nil { + panic(fmt.Errorf("reading cluster policy after snapshot install: %w", err)) + } + if err := capabilities.ValidateSupportedActive(policy, capabilities.SupportedIDs()); err != nil { + panic(fmt.Errorf("snapshot requires unsupported capability: %w", err)) + } + // Force-write the locally-authoritative self row to Pebble BEFORE // the reload, so LoadAll pulls our address rather than the leader's // potentially-stale view and the next checkpoint we serve is @@ -541,7 +643,8 @@ func (m *Membership) OnSnapshotInstalled() { // Better to fall through and let Rehydrate reload from Pebble // (with the leader's possibly-stale self); the next OnSnapshot // or restart will re-attempt the self-Put. - if err := m.store.Put(m.selfNodeID, m.selfRaftAddr, m.selfServiceAddr, m.selfInstanceID); err != nil { + selfCapabilities := capabilities.IDsAsUint32(capabilities.SupportedIDs()) + if err := m.store.Put(m.selfNodeID, m.selfRaftAddr, m.selfServiceAddr, m.selfInstanceID, selfCapabilities); err != nil { m.logger.WithFields(map[string]any{ "error": err, }).Errorf("Refreshing self in Pebble before snapshot reload failed; next checkpoint we serve may carry a stale self address") @@ -570,7 +673,21 @@ func (m *Membership) OnSnapshotInstalled() { // // Promotions repeat the existing peer registration, making replay independent // of the node-local membership cache. -func (m *Membership) WriteConfChange(entry *raftpb.Entry, session *dal.WriteSession) error { +// WriteConfChange uses the committed policy when no effective policy is given. +// An explicit policy, including nil, is the state at this entry's position in +// the FSM batch and avoids validating against an older committed policy. +func (m *Membership) WriteConfChange(entry *raftpb.Entry, session *dal.WriteSession, effectivePolicy ...*commonpb.ClusterPolicy) error { + if len(effectivePolicy) > 1 { + return errors.New("at most one effective policy may be supplied") + } + if len(effectivePolicy) == 1 { + return m.writeConfChange(entry, session, effectivePolicy[0], true) + } + + return m.writeConfChange(entry, session, nil, false) +} + +func (m *Membership) writeConfChange(entry *raftpb.Entry, session *dal.WriteSession, effectivePolicy *commonpb.ClusterPolicy, hasEffectivePolicy bool) error { cc, ok, err := UnmarshalConfChangeV2(entry) if err != nil { return fmt.Errorf("decoding ConfChange entry: %w", err) @@ -584,11 +701,43 @@ func (m *Membership) WriteConfChange(entry *raftpb.Entry, session *dal.WriteSess } writeRegistration := func(nodeID uint64, ctx *ConfChangeContext) error { + if err := ValidateCapabilities(ctx.Capabilities); err != nil { + return fmt.Errorf("invalid capability evidence for peer %d: %w", nodeID, err) + } + policy := effectivePolicy + if !hasEffectivePolicy { + m.mu.RLock() + provider := m.capabilityPolicyProvider + m.mu.RUnlock() + if provider != nil { + var err error + policy, err = provider() + if err != nil { + return fmt.Errorf("reading committed cluster policy for peer %d: %w", nodeID, err) + } + } + } + if policy != nil || hasEffectivePolicy { + required := make([]uint32, 0) + if policy != nil { + required = make([]uint32, 0, len(policy.GetCapabilityActivations())) + for _, activation := range policy.GetCapabilityActivations() { + if activation != nil && activation.GetActive() { + required = append(required, activation.GetId()) + } + } + } + slices.Sort(required) + if err := ValidateCapabilityEvidence(ctx, required); err != nil { + return fmt.Errorf("invalid capability evidence for peer %d: %w", nodeID, err) + } + } if err := session.SetProto(peerKey(nodeID), &raftcmdpb.PeerAddress{ NodeId: nodeID, RaftAddress: ctx.RaftAddress, ServiceAddress: ctx.ServiceAddress, InstanceId: ctx.InstanceID, + Capabilities: ctx.Capabilities, }); err != nil { return fmt.Errorf("session write peer %d: %w", nodeID, err) } diff --git a/internal/infra/membership/membership_test.go b/internal/infra/membership/membership_test.go index ea9e7587c4..9983de2990 100644 --- a/internal/infra/membership/membership_test.go +++ b/internal/infra/membership/membership_test.go @@ -11,6 +11,7 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" + "github.com/formancehq/ledger/v3/internal/proto/commonpb" "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" "github.com/formancehq/ledger/v3/internal/storage/dal" ) @@ -122,7 +123,7 @@ const ( func newTestMembership(t *testing.T) *Membership { t.Helper() - m, err := NewMembership(newTestPeerStore(t), noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(newTestPeerStore(t), noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) m.Start() @@ -135,9 +136,9 @@ func TestPeerStore_PutLoadAll(t *testing.T) { ps := newTestPeerStore(t) - require.NoError(t, ps.Put(1, "pod-0:7777", "pod-0:8888", fixedInstanceID(1))) - require.NoError(t, ps.Put(2, "pod-1:7777", "pod-1:8888", fixedInstanceID(2))) - require.NoError(t, ps.Put(3, "pod-2:7777", "pod-2:8888", fixedInstanceID(3))) + require.NoError(t, ps.Put(1, "pod-0:7777", "pod-0:8888", fixedInstanceID(1), nil)) + require.NoError(t, ps.Put(2, "pod-1:7777", "pod-1:8888", fixedInstanceID(2), nil)) + require.NoError(t, ps.Put(3, "pod-2:7777", "pod-2:8888", fixedInstanceID(3), nil)) peers, err := ps.LoadAll() require.NoError(t, err) @@ -152,8 +153,8 @@ func TestPeerStore_RejectsInvalidInstanceID(t *testing.T) { ps := newTestPeerStore(t) - require.ErrorContains(t, ps.Put(1, "pod-0:7777", "pod-0:8888", nil), "instance_id must be 16 bytes") - require.ErrorContains(t, ps.Put(1, "pod-0:7777", "pod-0:8888", []byte("short")), "instance_id must be 16 bytes") + require.ErrorContains(t, ps.Put(1, "pod-0:7777", "pod-0:8888", nil, nil), "instance_id must be 16 bytes") + require.ErrorContains(t, ps.Put(1, "pod-0:7777", "pod-0:8888", []byte("short"), nil), "instance_id must be 16 bytes") } func TestPeerStore_LoadAllRejectsPersistedInvalidInstanceID(t *testing.T) { @@ -188,6 +189,7 @@ func TestNewMembershipRejectsInvalidSelfInstanceID(t *testing.T) { testSelfServiceAddr, nil, logging.Testing(), + NoopClusterPolicyProvider, ) require.ErrorContains(t, err, "invalid self identity") } @@ -200,8 +202,8 @@ func TestPeerStore_PutOverwrites(t *testing.T) { ps := newTestPeerStore(t) - require.NoError(t, ps.Put(7, "old:7777", "old:8888", fixedInstanceID(7))) - require.NoError(t, ps.Put(7, "new:7777", "new:8888", fixedInstanceID(8))) + require.NoError(t, ps.Put(7, "old:7777", "old:8888", fixedInstanceID(7), nil)) + require.NoError(t, ps.Put(7, "new:7777", "new:8888", fixedInstanceID(8), nil)) peers, err := ps.LoadAll() require.NoError(t, err) @@ -216,8 +218,8 @@ func TestPeerStore_Delete(t *testing.T) { ps := newTestPeerStore(t) - require.NoError(t, ps.Put(1, "a:1", "a:2", fixedInstanceID(1))) - require.NoError(t, ps.Put(2, "b:1", "b:2", fixedInstanceID(2))) + require.NoError(t, ps.Put(1, "a:1", "a:2", fixedInstanceID(1), nil)) + require.NoError(t, ps.Put(2, "b:1", "b:2", fixedInstanceID(2), nil)) require.NoError(t, ps.Delete(1)) peers, err := ps.LoadAll() @@ -407,9 +409,10 @@ func TestMembership_OnSnapshotInstalled(t *testing.T) { ps := newTestPeerStore(t) // Pre-swap state: cluster A had peer 7. - require.NoError(t, ps.Put(7, "old:1", "old:2", fixedInstanceID(7))) + require.NoError(t, ps.Put(7, "old:1", "old:2", fixedInstanceID(7), nil)) + require.NoError(t, ps.Put(testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), []uint32{7})) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) m.Start() require.Equal(t, "old:1", m.PeerAddresses()[7].RaftAddress) @@ -417,8 +420,8 @@ func TestMembership_OnSnapshotInstalled(t *testing.T) { // Simulate a leader checkpoint restore: Pebble now has peers 1 + 3, // and no peer 7 (the source cluster A has been replaced by cluster B). require.NoError(t, ps.Delete(7)) - require.NoError(t, ps.Put(1, "new:1", "new:2", fixedInstanceID(1))) - require.NoError(t, ps.Put(3, "new:3", "new:4", fixedInstanceID(3))) + require.NoError(t, ps.Put(1, "new:1", "new:2", fixedInstanceID(1), nil)) + require.NoError(t, ps.Put(3, "new:3", "new:4", fixedInstanceID(3), nil)) // Hook fires: cache must catch up to the new Pebble state. m.OnSnapshotInstalled() @@ -430,18 +433,42 @@ func TestMembership_OnSnapshotInstalled(t *testing.T) { require.Equal(t, "new:1", got[1].RaftAddress) require.Equal(t, "new:3", got[3].RaftAddress) require.Equal(t, testSelfRaftAddr, got[testSelfNodeID].RaftAddress, "self must be re-upserted with local truth") + require.Empty(t, got[testSelfNodeID].Capabilities, "self evidence must reflect the running binary after snapshot restore") require.NotContains(t, got, uint64(7)) } +func TestMembershipRegisterSelfClearsStaleCapabilityEvidence(t *testing.T) { + ps := newTestPeerStore(t) + require.NoError(t, ps.Put(testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), []uint32{7})) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) + require.NoError(t, err) + + require.NoError(t, m.Register(testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), nil)) + peers, err := ps.LoadAll() + require.NoError(t, err) + require.Empty(t, peers[testSelfNodeID].Capabilities) + require.Empty(t, m.PeerAddresses()[testSelfNodeID].Capabilities) +} + +func TestMembershipOnSnapshotInstalledRejectsUnsupportedActiveCapability(t *testing.T) { + ps := newTestPeerStore(t) + provider := func() (*commonpb.ClusterPolicy, error) { + return &commonpb.ClusterPolicy{CapabilityActivations: []*commonpb.CapabilityActivation{{Id: 7, Active: true}}}, nil + } + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), provider) + require.NoError(t, err) + require.PanicsWithError(t, "snapshot requires unsupported capability: active capability 7 is not supported by this binary", m.OnSnapshotInstalled) +} + func TestMembership_SnapshotDonorCandidates(t *testing.T) { t.Parallel() ps := newTestPeerStore(t) - require.NoError(t, ps.Put(9, "peer-9:7000", "peer-9:8000", fixedInstanceID(9))) - require.NoError(t, ps.Put(2, "peer-2:7000", "peer-2:8000", fixedInstanceID(2))) - require.NoError(t, ps.Put(testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42))) + require.NoError(t, ps.Put(9, "peer-9:7000", "peer-9:8000", fixedInstanceID(9), nil)) + require.NoError(t, ps.Put(2, "peer-2:7000", "peer-2:8000", fixedInstanceID(2), nil)) + require.NoError(t, ps.Put(testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), nil)) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) require.Equal(t, []uint64{2, 9}, m.SnapshotDonorCandidates()) @@ -451,7 +478,7 @@ func TestMembership_OnSnapshotInstalledPanicsOnInvalidPeerIdentity(t *testing.T) t.Parallel() ps := newTestPeerStore(t) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) // Simulate a leader checkpoint that replaced Pebble with an invalid peer @@ -486,12 +513,12 @@ func TestMembership_RehydrateAfterReplay(t *testing.T) { ps := newTestPeerStore(t) - require.NoError(t, ps.Put(7, "before:1", "before:2", fixedInstanceID(7))) - require.NoError(t, ps.Put(9, "removed:1", "removed:2", fixedInstanceID(9))) + require.NoError(t, ps.Put(7, "before:1", "before:2", fixedInstanceID(7), nil)) + require.NoError(t, ps.Put(9, "removed:1", "removed:2", fixedInstanceID(9), nil)) raftTransport := &recordingTransport{peers: map[uint64]string{}} servicePool := &recordingPool{peers: map[uint64]string{}} - m, err := NewMembership(ps, raftTransport, servicePool, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, raftTransport, servicePool, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) m.Start() require.Equal(t, "before:1", m.PeerAddresses()[7].RaftAddress) @@ -502,10 +529,10 @@ func TestMembership_RehydrateAfterReplay(t *testing.T) { // Simulate WAL replay: WriteConfChange wrote these rows directly to // Pebble without touching the cache. - require.NoError(t, ps.Put(7, "after:7", "after:8", fixedInstanceID(7))) + require.NoError(t, ps.Put(7, "after:7", "after:8", fixedInstanceID(7), nil)) require.NoError(t, ps.Delete(9)) - require.NoError(t, ps.Put(1, "after:1", "after:2", fixedInstanceID(1))) - require.NoError(t, ps.Put(3, "after:3", "after:4", fixedInstanceID(3))) + require.NoError(t, ps.Put(1, "after:1", "after:2", fixedInstanceID(1), nil)) + require.NoError(t, ps.Put(3, "after:3", "after:4", fixedInstanceID(3), nil)) require.NoError(t, m.Rehydrate()) @@ -541,15 +568,15 @@ func TestMembership_StartGate(t *testing.T) { pool := &countingPool{} ps := newTestPeerStore(t) - require.NoError(t, ps.Put(1, "pod-0:7777", "pod-0:8888", fixedInstanceID(1))) + require.NoError(t, ps.Put(1, "pod-0:7777", "pod-0:8888", fixedInstanceID(1), nil)) - m, err := NewMembership(ps, transport, pool, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, transport, pool, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) require.Equal(t, 0, transport.adds, "no wire before Start (cache-only construction)") require.Equal(t, 0, pool.adds, "no wire before Start (cache-only construction)") - require.NoError(t, m.Set(2, "pod-1:7777", "pod-1:8888", fixedInstanceID(2))) + require.NoError(t, m.Set(2, "pod-1:7777", "pod-1:8888", fixedInstanceID(2), nil)) require.Equal(t, 0, transport.adds, "Set pre-Start must be cache-only") require.Equal(t, 0, pool.adds, "Set pre-Start must be cache-only") require.Len(t, m.PeerAddresses(), 2, "cache must still reflect the Set") @@ -558,7 +585,7 @@ func TestMembership_StartGate(t *testing.T) { require.Equal(t, 2, transport.adds, "Start flushes the cache to the transport") require.Equal(t, 2, pool.adds, "Start flushes the cache to the service pool") - require.NoError(t, m.Set(3, "pod-2:7777", "pod-2:8888", fixedInstanceID(3))) + require.NoError(t, m.Set(3, "pod-2:7777", "pod-2:8888", fixedInstanceID(3), nil)) require.Equal(t, 3, transport.adds, "Set post-Start wires inline") require.Equal(t, 3, pool.adds, "Set post-Start wires inline") @@ -571,7 +598,7 @@ func TestMembership_SetRejectsInvalidInstanceID(t *testing.T) { t.Parallel() m := newTestMembership(t) - require.ErrorContains(t, m.Set(1, "pod-0:7777", "pod-0:8888", nil), "instance_id must be 16 bytes") + require.ErrorContains(t, m.Set(1, "pod-0:7777", "pod-0:8888", nil, nil), "instance_id must be 16 bytes") require.NotContains(t, m.PeerAddresses(), uint64(1)) } @@ -589,15 +616,16 @@ func TestMembership_SetRewiresChangedAddresses(t *testing.T) { testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), + NoopClusterPolicyProvider, ) require.NoError(t, err) m.Start() - require.NoError(t, m.Set(1, "old:7777", "old:8888", fixedInstanceID(1))) + require.NoError(t, m.Set(1, "old:7777", "old:8888", fixedInstanceID(1), nil)) require.Equal(t, 1, transport.adds) require.Equal(t, 0, transport.removes) - require.NoError(t, m.Set(1, "new:7777", "new:8888", fixedInstanceID(2))) + require.NoError(t, m.Set(1, "new:7777", "new:8888", fixedInstanceID(2), nil)) require.Equal(t, 2, transport.adds) // wireRemove is intentionally not called on a Raft-address change: DefaultTransport.AddPeer // refreshes the pooled connection address through ConnectionPool.AddPeer while preserving the @@ -620,12 +648,12 @@ func TestMembership_SetKeepsRaftPeerOnServiceAddressChange(t *testing.T) { transport := &countingTransport{} pool := &countingPool{} m, err := NewMembership(newTestPeerStore(t), transport, pool, - testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) m.Start() - require.NoError(t, m.Set(1, "raft:7777", "old:8888", fixedInstanceID(1))) - require.NoError(t, m.Set(1, "raft:7777", "new:8888", fixedInstanceID(1))) + require.NoError(t, m.Set(1, "raft:7777", "old:8888", fixedInstanceID(1), nil)) + require.NoError(t, m.Set(1, "raft:7777", "new:8888", fixedInstanceID(1), nil)) // Removing the existing Raft peer would require a fresh optional-TLS probe; // a failed probe could then strand a previously healthy replication channel. @@ -641,7 +669,7 @@ func TestMembership_ClonesInstanceIDs(t *testing.T) { m := newTestMembership(t) instanceID := fixedInstanceID(1) - require.NoError(t, m.Set(1, "pod-0:7777", "pod-0:8888", instanceID)) + require.NoError(t, m.Set(1, "pod-0:7777", "pod-0:8888", instanceID, nil)) instanceID[0] = 9 stored, ok := m.GetInstanceID(1) @@ -660,6 +688,20 @@ func TestMembership_ClonesInstanceIDs(t *testing.T) { require.Equal(t, byte(1), again[0], "PeerAddresses must deep-clone instance IDs") } +func TestPeerStore_PersistsCapabilityEvidenceWithIdentity(t *testing.T) { + ps := newTestPeerStore(t) + caps := []uint32{1, 2} + identity := fixedInstanceID(0x7a) + require.NoError(t, ps.Put(9, "raft:9", "service:9", identity, caps)) + + rows, err := ps.LoadAll() + require.NoError(t, err) + row, ok := rows[9] + require.True(t, ok) + require.Equal(t, identity, row.InstanceID) + require.Equal(t, caps, row.Capabilities) +} + // TestMembership_ReconcileAgainstConfState pins the boot-time cleanup // pass that drops peers whose NodeID is no longer in the durable // ConfState. This covers two crash windows: interrupted ForceRemoveNode @@ -672,13 +714,13 @@ func TestMembership_ReconcileAgainstConfState(t *testing.T) { newSeeded := func(t *testing.T) *Membership { ps := newTestPeerStore(t) - require.NoError(t, ps.Put(7, "self:1", "self:2", fixedInstanceID(7))) - require.NoError(t, ps.Put(1, "voter1:1", "voter1:2", fixedInstanceID(1))) - require.NoError(t, ps.Put(2, "voter2:1", "voter2:2", fixedInstanceID(2))) - require.NoError(t, ps.Put(3, "learner:1", "learner:2", fixedInstanceID(3))) - require.NoError(t, ps.Put(99, "stale:1", "stale:2", fixedInstanceID(99))) + require.NoError(t, ps.Put(7, "self:1", "self:2", fixedInstanceID(7), nil)) + require.NoError(t, ps.Put(1, "voter1:1", "voter1:2", fixedInstanceID(1), nil)) + require.NoError(t, ps.Put(2, "voter2:1", "voter2:2", fixedInstanceID(2), nil)) + require.NoError(t, ps.Put(3, "learner:1", "learner:2", fixedInstanceID(3), nil)) + require.NoError(t, ps.Put(99, "stale:1", "stale:2", fixedInstanceID(99), nil)) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, 7, "self:1", "self:2", fixedInstanceID(7), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, 7, "self:1", "self:2", fixedInstanceID(7), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) m.Start() diff --git a/internal/infra/membership/peer_store.go b/internal/infra/membership/peer_store.go index fd3797e738..0b9314d4d3 100644 --- a/internal/infra/membership/peer_store.go +++ b/internal/infra/membership/peer_store.go @@ -91,10 +91,17 @@ func peerKeyRange() (lower, upper []byte) { // from lifecycle paths for self and initial-peer registration. ConfChange // apply uses the caller's WriteSession directly. Every persisted peer must // carry the concrete member's 16-byte WAL/PVC identity. -func (p *PeerStore) Put(nodeID uint64, raftAddr, serviceAddr string, instanceID []byte) error { +// Put persists a peer directory row together with the capability +// evidence bound to its concrete instance. The evidence is deliberately part +// of the same Pebble value as the identity so a crash cannot expose a peer +// with a fresh instance_id and stale capabilities. +func (p *PeerStore) Put(nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilities []uint32) error { if err := ValidateInstanceID(instanceID); err != nil { return fmt.Errorf("writing peer %d: %w", nodeID, err) } + if err := ValidateCapabilities(capabilities); err != nil { + return fmt.Errorf("writing peer %d: %w", nodeID, err) + } session := p.store.OpenWriteSession() if err := session.SetProto(peerKey(nodeID), &raftcmdpb.PeerAddress{ @@ -102,6 +109,7 @@ func (p *PeerStore) Put(nodeID uint64, raftAddr, serviceAddr string, instanceID RaftAddress: raftAddr, ServiceAddress: serviceAddr, InstanceId: instanceID, + Capabilities: capabilities, }); err != nil { _ = session.Cancel() @@ -182,6 +190,7 @@ func (p *PeerStore) LoadAll() (map[uint64]ConfChangeContext, error) { RaftAddress: pa.GetRaftAddress(), ServiceAddress: pa.GetServiceAddress(), InstanceID: pa.GetInstanceId(), + Capabilities: append([]uint32(nil), pa.GetCapabilities()...), } } diff --git a/internal/infra/membership/remove_node_blacklist_test.go b/internal/infra/membership/remove_node_blacklist_test.go index 8ec6056cc0..9cd9dfe582 100644 --- a/internal/infra/membership/remove_node_blacklist_test.go +++ b/internal/infra/membership/remove_node_blacklist_test.go @@ -52,9 +52,9 @@ func TestWriteConfChange_RemoveNodeBlacklistsPeer(t *testing.T) { ps, store := storesForBlacklistTest(t) instanceID := fixedInstanceID(0xAA) - require.NoError(t, ps.Put(3, "pod-2:7777", "pod-2:8888", instanceID)) + require.NoError(t, ps.Put(3, "pod-2:7777", "pod-2:8888", instanceID, nil)) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) ccCtx, err := MarshalConfChangeContext(ConfChangeContext{InstanceID: instanceID}) @@ -93,9 +93,9 @@ func TestWriteConfChange_RemoveNodeWithoutIdentityFailsLoudly(t *testing.T) { ps, store := storesForBlacklistTest(t) instanceID := fixedInstanceID(0xAA) - require.NoError(t, ps.Put(3, "pod-2:7777", "pod-2:8888", instanceID)) + require.NoError(t, ps.Put(3, "pod-2:7777", "pod-2:8888", instanceID, nil)) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) ccCtx, err := MarshalConfChangeContext(ConfChangeContext{ProposalID: "remove-3"}) require.NoError(t, err) @@ -133,7 +133,7 @@ func TestWriteConfChange_AddLearnerWithoutIdentityFailsLoudly(t *testing.T) { t.Parallel() ps, store := storesForBlacklistTest(t) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) cc := &raftpb.ConfChangeV2{ @@ -164,9 +164,9 @@ func TestUnregisterAndBlacklist_AtomicBatch(t *testing.T) { ps, _ := storesForBlacklistTest(t) instanceID := fixedInstanceID(0xBB) - require.NoError(t, ps.Put(7, "pod-6:7777", "pod-6:8888", instanceID)) + require.NoError(t, ps.Put(7, "pod-6:7777", "pod-6:8888", instanceID, nil)) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) require.NoError(t, m.UnregisterAndBlacklist(7, instanceID, 42)) @@ -187,7 +187,7 @@ func TestUnregisterAndBlacklist_RejectsWrongInstanceIDLen(t *testing.T) { ps, _ := storesForBlacklistTest(t) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) require.Error(t, m.UnregisterAndBlacklist(7, []byte("short"), 0)) @@ -202,7 +202,7 @@ func TestIsRemoved_MatchesOnExactTuple(t *testing.T) { ps, _ := storesForBlacklistTest(t) blacklisted := fixedInstanceID(0xCC) - m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing()) + m, err := NewMembership(ps, noopTransport{}, noopPool{}, testSelfNodeID, testSelfRaftAddr, testSelfServiceAddr, fixedInstanceID(0x42), logging.Testing(), NoopClusterPolicyProvider) require.NoError(t, err) require.NoError(t, m.UnregisterAndBlacklist(9, blacklisted, 100)) diff --git a/internal/infra/node/add_learner_classify_test.go b/internal/infra/node/add_learner_classify_test.go index 48ecbca310..4095a82ca6 100644 --- a/internal/infra/node/add_learner_classify_test.go +++ b/internal/infra/node/add_learner_classify_test.go @@ -11,8 +11,8 @@ func TestAddLearnerPathsRejectInvalidInstanceIDBeforeRaftAccess(t *testing.T) { t.Parallel() n := &Node{} - require.ErrorContains(t, n.AddLearner(context.Background(), 1, "r:1", "s:1", nil), "instance_id must be 16 bytes") - require.ErrorContains(t, n.JoinAsLearner(context.Background(), 1, "r:1", "s:1", []byte("short")), "instance_id must be 16 bytes") + require.ErrorContains(t, n.AddLearner(context.Background(), 1, "r:1", "s:1", nil, nil), "instance_id must be 16 bytes") + require.ErrorContains(t, n.JoinAsLearner(context.Background(), 1, "r:1", "s:1", []byte("short"), nil), "instance_id must be 16 bytes") } // TestClassifyExistingLearner locks in the EN-1436 fail-fast decision. diff --git a/internal/infra/node/applier_async_storage_test.go b/internal/infra/node/applier_async_storage_test.go index 8cb5fbf273..9a263cb7c9 100644 --- a/internal/infra/node/applier_async_storage_test.go +++ b/internal/infra/node/applier_async_storage_test.go @@ -165,6 +165,7 @@ func TestApplierNewApplierRejectsNilSink(t *testing.T) { logger, nodeRegistry, nodeSnapshotter, pebbleStore, dal.NewSentinelFactory(pebbleStore, false), meterProvider, nil, state.NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, + state.NoopCapabilityPolicyCommitHandler, ) require.NoError(t, err) diff --git a/internal/infra/node/applier_test.go b/internal/infra/node/applier_test.go index d7427c9869..a41dcf3660 100644 --- a/internal/infra/node/applier_test.go +++ b/internal/infra/node/applier_test.go @@ -167,7 +167,7 @@ func newTestApplierSetupWithHandlers(t *testing.T, sink LocalResponses, notifier logger, nodeRegistry, nodeSnapshotter, pebbleStore, dal.NewSentinelFactory(pebbleStore, false), meterProvider, nil, sharedState, notifier, nil, "test-cluster", 0, handler, - ) + state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) recovery := state.NewRecovery(fsm, pebbleStore) diff --git a/internal/infra/node/capability_membership_test.go b/internal/infra/node/capability_membership_test.go new file mode 100644 index 0000000000..cccb1aa38d --- /dev/null +++ b/internal/infra/node/capability_membership_test.go @@ -0,0 +1,41 @@ +package node + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/formancehq/ledger/v3/internal/infra/state" + "github.com/formancehq/ledger/v3/internal/pkg/futures" +) + +func TestCapabilityMembershipFenceHeldUntilPolicyApply(t *testing.T) { + n := &Node{stopChannel: make(chan struct{}), terminalCh: make(chan struct{})} + n.confChangeMu.Configure(nil, n.stopChannel, n.terminalCh) + release, err := n.confChangeMu.AcquireClusterPolicy(context.Background()) + require.NoError(t, err) + future := futures.New[state.ApplyResult]() + release(future) + require.False(t, n.confChangeMu.TryLock(), "a learner must not pass a pending activation") + + future.Resolve(state.ApplyResult{}, nil) + require.Eventually(t, func() bool { + if !n.confChangeMu.TryLock() { + return false + } + n.confChangeMu.Unlock() + + return true + }, time.Second, time.Millisecond) +} + +func TestCapabilityMembershipFenceReleasedWithoutProposal(t *testing.T) { + n := &Node{} + release, err := n.confChangeMu.AcquireClusterPolicy(context.Background()) + require.NoError(t, err) + release(nil) + require.True(t, n.confChangeMu.TryLock()) + n.confChangeMu.Unlock() +} diff --git a/internal/infra/node/conf_change_retry_durable_test.go b/internal/infra/node/conf_change_retry_durable_test.go index f8860b81af..1b37d844ce 100644 --- a/internal/infra/node/conf_change_retry_durable_test.go +++ b/internal/infra/node/conf_change_retry_durable_test.go @@ -11,6 +11,7 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" + "github.com/formancehq/ledger/v3/internal/storage/dal" "github.com/formancehq/ledger/v3/internal/storage/wal" ) @@ -29,7 +30,9 @@ func testPromoteLearnerRetryDuringWALPersistence(t *testing.T, cancelDuringWAL b n := newConfiguredPeersTestNode(t) n.logger = logging.Testing() - setup := newTestApplierSetupWithConfChangeHandler(t, make(LocalResponses, 1024), n.membership.WriteConfChange) + setup := newTestApplierSetupWithConfChangeHandler(t, make(LocalResponses, 1024), func(entry *raftpb.Entry, session *dal.WriteSession) error { + return n.membership.WriteConfChange(entry, session) + }) n.fsm = setup.fsm waitCtx, cancelWait := context.WithTimeout(t.Context(), 10*time.Second) defer cancelWait() diff --git a/internal/infra/node/confchange_identity_test.go b/internal/infra/node/confchange_identity_test.go index 137df7e110..6808bd29b8 100644 --- a/internal/infra/node/confchange_identity_test.go +++ b/internal/infra/node/confchange_identity_test.go @@ -58,7 +58,7 @@ func TestPromotionContextCarriesCompleteRegisteredIdentity(t *testing.T) { n := &Node{membership: m} _, err := n.promotionContext(9, "promotion") require.ErrorContains(t, err, "learner 9 has no membership row") - require.NoError(t, m.Set(9, "peer:7777", "peer:8888", []byte("peer-instance-id"))) + require.NoError(t, m.Set(9, "peer:7777", "peer:8888", []byte("peer-instance-id"), nil)) for _, proposalID := range []string{"", "promotion"} { payload, err := n.promotionContext(9, proposalID) require.NoError(t, err) @@ -81,7 +81,7 @@ func TestPromotionContextRejectsIncompleteAddresses(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() m := newTestMembership(t) - require.NoError(t, m.Set(9, addresses[0], addresses[1], []byte("peer-instance-id"))) + require.NoError(t, m.Set(9, addresses[0], addresses[1], []byte("peer-instance-id"), nil)) n := &Node{membership: m} payload, err := n.promotionContext(9, "promotion") require.Nil(t, payload) diff --git a/internal/infra/node/config.go b/internal/infra/node/config.go index f91fbe6d59..4c5b91c1c1 100644 --- a/internal/infra/node/config.go +++ b/internal/infra/node/config.go @@ -177,7 +177,8 @@ func (cfg *NodeConfig) SetDefaults() { type Peer struct { ID uint64 - Address string // Raft transport address - ServiceAddress string // Service API address (for request forwarding) - InstanceID []byte // 16-byte WAL/PVC incarnation identity + Address string // Raft transport address + ServiceAddress string // Service API address (for request forwarding) + InstanceID []byte // 16-byte WAL/PVC incarnation identity + Capabilities []uint32 // capabilities supported by this binary } diff --git a/internal/infra/node/configured_peers_test.go b/internal/infra/node/configured_peers_test.go index 58edea25fa..bd80a89d53 100644 --- a/internal/infra/node/configured_peers_test.go +++ b/internal/infra/node/configured_peers_test.go @@ -41,10 +41,10 @@ func newConfiguredPeersTestNode(t *testing.T) *Node { } require.Equal(t, raft.StateLeader, rawNode.Status().RaftState) m := newTestMembership(t) - require.NoError(t, m.Set(1, "self:7777", "self:8888", []byte("self-instance-id"))) - require.NoError(t, m.Set(2, "old:7777", "old:8888", []byte("peer-instance-id"))) + require.NoError(t, m.Set(1, "self:7777", "self:8888", []byte("self-instance-id"), nil)) + require.NoError(t, m.Set(2, "old:7777", "old:8888", []byte("peer-instance-id"), nil)) // A cached row alone is not evidence of a configured member. - require.NoError(t, m.Set(3, "extra:7777", "extra:8888", []byte("extra-instanceid"))) + require.NoError(t, m.Set(3, "extra:7777", "extra:8888", []byte("extra-instanceid"), nil)) return &Node{ rawNode: rawNode, @@ -113,7 +113,7 @@ func TestGetConfiguredPeers_SnapshotSurvivesMembershipChange(t *testing.T) { n.rawNode.ApplyConfChange(&raftpb.ConfChangeV2{Changes: []*raftpb.ConfChangeSingle{{ Type: new(raftpb.ConfChangeAddLearnerNode), NodeId: new(uint64(2)), }}}) - require.NoError(t, n.membership.Set(2, "new:7777", "new:8888", []byte("next-instance-id"))) + require.NoError(t, n.membership.Set(2, "new:7777", "new:8888", []byte("next-instance-id"), nil)) } cmd.errCh <- err got := <-results @@ -168,7 +168,7 @@ func TestGetConfiguredPeers_TopologySnapshotIsAtomicWithConcurrentRehydrate(t *t // the snapshot, never in between. setDone := make(chan struct{}) go func() { - require.NoError(t, n.membership.Set(2, "new:7777", "new:8888", []byte("next-instance-id"))) + require.NoError(t, n.membership.Set(2, "new:7777", "new:8888", []byte("next-instance-id"), nil)) close(setDone) }() err := cmd.fn() @@ -211,10 +211,10 @@ func TestGetConfiguredPeers_TopologySnapshotIsAtomicWithConcurrentRehydrate_Dete // Persist all configured members to Pebble so Rehydrate does not drop // voter 1 from the cache. Set() updates only the in-memory cache; // Rehydrate replaces the whole cache from LoadAll. - require.NoError(t, n.membership.PeerStore().Put(1, "self:7777", "self:8888", []byte("self-instance-id"))) + require.NoError(t, n.membership.PeerStore().Put(1, "self:7777", "self:8888", []byte("self-instance-id"), nil)) // Persist the new incarnation for peer 2 so Rehydrate loads it. // The in-memory Membership cache still holds the old row. - require.NoError(t, n.membership.PeerStore().Put(2, "new:7777", "new:8888", []byte("next-instance-id"))) + require.NoError(t, n.membership.PeerStore().Put(2, "new:7777", "new:8888", []byte("next-instance-id"), nil)) type result struct { peers []Peer err error @@ -276,8 +276,8 @@ func TestGetConfiguredPeers_RehydrateBeforeSnapshotIsVisibleInCapture(t *testing // Persist all configured members to Pebble before Rehydrate. // Set() updates only the in-memory cache; Rehydrate reloads from Pebble. // Both voter-1 (self) and the updated peer-2 row must be present. - require.NoError(t, n.membership.PeerStore().Put(1, "self:7777", "self:8888", []byte("self-instance-id"))) - require.NoError(t, n.membership.PeerStore().Put(2, "new:7777", "new:8888", []byte("next-instance-id"))) + require.NoError(t, n.membership.PeerStore().Put(1, "self:7777", "self:8888", []byte("self-instance-id"), nil)) + require.NoError(t, n.membership.PeerStore().Put(2, "new:7777", "new:8888", []byte("next-instance-id"), nil)) require.NoError(t, n.membership.Rehydrate()) results := make(chan []Peer, 1) diff --git a/internal/infra/node/delete_ledger_sentinel_recovery_test.go b/internal/infra/node/delete_ledger_sentinel_recovery_test.go index 6b6e919aaf..9911318a3a 100644 --- a/internal/infra/node/delete_ledger_sentinel_recovery_test.go +++ b/internal/infra/node/delete_ledger_sentinel_recovery_test.go @@ -173,7 +173,7 @@ func newSentinelApplier(t *testing.T, dir string, sentinelMode bool) (*testAppli registry := state.NewStateRegistry(c, attributes.New()) fsm, err := state.NewMachine(logger, registry, state.NewCacheSnapshotter(logger, registry, nil), s, dal.NewSentinelFactory(s, sentinelMode), provider, nil, state.NewSharedState(), newNoopNotifier(t), nil, - "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }) + "", 0, func(*raftpb.Entry, *dal.WriteSession) error { return nil }, state.NoopCapabilityPolicyCommitHandler) require.NoError(t, err) recovery := state.NewRecovery(fsm, s) require.NoError(t, recovery.RecoverState()) diff --git a/internal/infra/node/drain.go b/internal/infra/node/drain.go index ec0758f771..41edf53013 100644 --- a/internal/infra/node/drain.go +++ b/internal/infra/node/drain.go @@ -49,11 +49,10 @@ func (node *Node) waitForPriorMembership(ctx context.Context) error { } cmd := commands.NewCommand() cmd.CallerSnapshot = commands.SystemCallerSnapshot(commands.ComponentClusterPeer) - data, err := cmd.MarshalVT() + proposal, err := NewProposal(cmd.GetId(), cmd) if err != nil { - return fmt.Errorf("marshaling membership barrier: %w", err) + return fmt.Errorf("preparing membership barrier: %w", err) } - proposal := NewProposal(cmd.GetId(), data) proposal.membershipBarrier = true node.indexTracker.Lock() future, err := node.Propose(ctx, proposal) @@ -114,11 +113,10 @@ func (node *Node) BeginDrain(ctx context.Context, epoch string, members []uint64 Kind: &raftcmdpb.TechnicalUpdate_DrainMarker{DrainMarker: &raftcmdpb.DrainMarker{Epoch: epoch, Members: append([]uint64(nil), members...)}}, }) cmd.CallerSnapshot = commands.SystemCallerSnapshot(commands.ComponentClusterPeer) - data, err := cmd.MarshalVT() + proposal, err := NewProposal(cmd.GetId(), cmd) if err != nil { - return nil, fmt.Errorf("marshaling drain marker: %w", err) + return nil, fmt.Errorf("preparing drain marker: %w", err) } - proposal := NewProposal(cmd.GetId(), data) proposal.drainMarker = true proposal.expectedTerm = expectedTerm @@ -178,11 +176,10 @@ func (node *Node) EndDrain(ctx context.Context, epoch string, expectedTerm uint6 Kind: &raftcmdpb.TechnicalUpdate_DrainRelease{DrainRelease: &raftcmdpb.DrainRelease{Epoch: epoch}}, }) cmd.CallerSnapshot = commands.SystemCallerSnapshot(commands.ComponentClusterPeer) - data, err := cmd.MarshalVT() + proposal, err := NewProposal(cmd.GetId(), cmd) if err != nil { - return nil, fmt.Errorf("marshaling drain release: %w", err) + return nil, fmt.Errorf("preparing drain release: %w", err) } - proposal := NewProposal(cmd.GetId(), data) proposal.releaseMarker = true proposal.expectedTerm = expectedTerm node.indexTracker.Lock() diff --git a/internal/infra/node/drain_test.go b/internal/infra/node/drain_test.go index 4066cdc96b..585a9f3fd4 100644 --- a/internal/infra/node/drain_test.go +++ b/internal/infra/node/drain_test.go @@ -84,7 +84,7 @@ func TestProposeRejectsAfterDrainEnqueue(t *testing.T) { t.Parallel() n, _ := newTestMaintenanceNode(t) n.drainOutstanding = 1 - _, err := n.Propose(t.Context(), NewProposal(1, []byte("business"))) + _, err := n.Propose(t.Context(), newProposal(1, []byte("business"), nil)) require.ErrorIs(t, err, domain.ErrMaintenanceMode) } @@ -130,7 +130,7 @@ func TestDrainProposalRejectsPreflightFromPriorLeadershipTerm(t *testing.T) { require.Contains(t, rn.Status().Progress, uint64(2)) require.False(t, rn.HasReady()) - marker := NewProposal(1, []byte("stale drain marker for member 1")) + marker := newProposal(1, []byte("stale drain marker for member 1"), nil) marker.drainMarker = true marker.expectedTerm = preflightTerm n.indexTracker.Lock() @@ -146,7 +146,7 @@ func TestDrainProposalRejectsPreflightFromPriorLeadershipTerm(t *testing.T) { require.False(t, n.drainActiveOrPending()) require.False(t, rn.HasReady(), "stale marker must not enter the Raft log") - release := NewProposal(2, []byte("stale release marker")) + release := newProposal(2, []byte("stale release marker"), nil) release.releaseMarker = true release.expectedTerm = preflightTerm n.indexTracker.Lock() @@ -167,7 +167,7 @@ func TestMembershipFenceOutlivesTimedOutDrainCaller(t *testing.T) { n, setup := newTestMaintenanceNode(t) n.proposeCh = make(chan *Proposal, 1) n.indexTracker = NewIndexTracker(1) - marker := NewProposal(1, []byte("drain marker")) + marker := newProposal(1, []byte("drain marker"), nil) marker.drainMarker = true n.indexTracker.Lock() future, err := n.Propose(t.Context(), marker) @@ -206,7 +206,7 @@ func TestReleaseMarkerCanCrossClosedAdmission(t *testing.T) { n.proposeCh = make(chan *Proposal, 1) n.indexTracker = NewIndexTracker(1) n.drainOutstanding = 1 - release := NewProposal(1, []byte("release")) + release := newProposal(1, []byte("release"), nil) release.releaseMarker = true n.indexTracker.Lock() _, err := n.Propose(t.Context(), release) @@ -219,20 +219,20 @@ func TestDroppedDrainMarkerReopensAdmission(t *testing.T) { n, _ := newTestMaintenanceNode(t) n.proposeCh = make(chan *Proposal, 3) n.indexTracker = NewIndexTracker(1) - first := NewProposal(1, []byte("first marker")) + first := newProposal(1, []byte("first marker"), nil) first.drainMarker = true n.indexTracker.Lock() _, err := n.Propose(t.Context(), first) n.indexTracker.Unlock() require.NoError(t, err) - second := NewProposal(2, []byte("second marker")) + second := newProposal(2, []byte("second marker"), nil) second.drainMarker = true n.indexTracker.Lock() secondFuture, err := n.Propose(t.Context(), second) n.indexTracker.Unlock() require.NoError(t, err) n.applier.ResolveDroppedFuture(first.commandID, ErrNotLeader) - _, err = n.Propose(t.Context(), NewProposal(1, []byte("still blocked"))) + _, err = n.Propose(t.Context(), newProposal(1, []byte("still blocked"), nil)) require.ErrorIs(t, err, domain.ErrMaintenanceMode) // The original RPC may have timed out. The applier still owns the // callback, so a later-term truncation releases the remaining slot. @@ -244,7 +244,7 @@ func TestDroppedDrainMarkerReopensAdmission(t *testing.T) { _, err = secondFuture.Wait(t.Context()) require.ErrorIs(t, err, ErrLeadershipLost) n.indexTracker.Lock() - _, err = n.Propose(t.Context(), NewProposal(3, []byte("admitted after definitive drops"))) + _, err = n.Propose(t.Context(), newProposal(3, []byte("admitted after definitive drops"), nil)) n.indexTracker.Unlock() require.NoError(t, err) } @@ -254,7 +254,7 @@ func TestRejectedQueuedDrainMarkerReopensAdmission(t *testing.T) { n, _ := newTestMaintenanceNode(t) n.proposeCh = make(chan *Proposal, 2) n.indexTracker = NewIndexTracker(1) - marker := NewProposal(1, []byte("drain marker")) + marker := newProposal(1, []byte("drain marker"), nil) marker.drainMarker = true n.indexTracker.Lock() _, err := n.Propose(t.Context(), marker) @@ -263,7 +263,7 @@ func TestRejectedQueuedDrainMarkerReopensAdmission(t *testing.T) { require.Same(t, marker, <-n.proposeCh) n.rejectQueuedProposal(marker, ErrNotLeader) n.indexTracker.Lock() - _, err = n.Propose(t.Context(), NewProposal(2, []byte("business after drop"))) + _, err = n.Propose(t.Context(), newProposal(2, []byte("business after drop"), nil)) n.indexTracker.Unlock() require.NoError(t, err) } @@ -273,18 +273,18 @@ func TestDrainEnqueueOrdersInFlightProposals(t *testing.T) { n, _ := newTestMaintenanceNode(t) n.proposeCh = make(chan *Proposal, 3) n.indexTracker = NewIndexTracker(1) - pre := NewProposal(1, []byte("already admitted")) + pre := newProposal(1, []byte("already admitted"), nil) n.indexTracker.Lock() _, err := n.Propose(t.Context(), pre) n.indexTracker.Unlock() require.NoError(t, err) - marker := NewProposal(2, []byte("drain marker")) + marker := newProposal(2, []byte("drain marker"), nil) marker.drainMarker = true n.indexTracker.Lock() _, err = n.Propose(t.Context(), marker) n.indexTracker.Unlock() require.NoError(t, err) - _, err = n.Propose(t.Context(), NewProposal(3, []byte("later write"))) + _, err = n.Propose(t.Context(), newProposal(3, []byte("later write"), nil)) require.ErrorIs(t, err, domain.ErrMaintenanceMode) require.Same(t, pre, <-n.proposeCh) require.Same(t, marker, <-n.proposeCh) @@ -296,25 +296,25 @@ func TestOldDrainDropCannotOpenNewEpoch(t *testing.T) { n, setup := newTestMaintenanceNode(t) n.proposeCh = make(chan *Proposal, 3) n.indexTracker = NewIndexTracker(1) - old := NewProposal(1, []byte("old marker")) + old := newProposal(1, []byte("old marker"), nil) old.drainMarker = true n.indexTracker.Lock() _, err := n.Propose(t.Context(), old) n.indexTracker.Unlock() require.NoError(t, err) setup.sharedState.SetDrainReleased(7) - newMarker := NewProposal(2, []byte("new marker")) + newMarker := newProposal(2, []byte("new marker"), nil) newMarker.drainMarker = true n.indexTracker.Lock() _, err = n.Propose(t.Context(), newMarker) n.indexTracker.Unlock() require.NoError(t, err) n.applier.ResolveDroppedFuture(old.commandID, ErrNotLeader) - _, err = n.Propose(t.Context(), NewProposal(3, []byte("must stay closed"))) + _, err = n.Propose(t.Context(), newProposal(3, []byte("must stay closed"), nil)) require.ErrorIs(t, err, domain.ErrMaintenanceMode) n.applier.ResolveDroppedFuture(newMarker.commandID, ErrNotLeader) n.indexTracker.Lock() - _, err = n.Propose(t.Context(), NewProposal(4, []byte("new marker dropped"))) + _, err = n.Propose(t.Context(), newProposal(4, []byte("new marker dropped"), nil)) n.indexTracker.Unlock() require.NoError(t, err) } diff --git a/internal/infra/node/force_remove_durability_test.go b/internal/infra/node/force_remove_durability_test.go index e58e4a23e0..582a192447 100644 --- a/internal/infra/node/force_remove_durability_test.go +++ b/internal/infra/node/force_remove_durability_test.go @@ -216,6 +216,7 @@ func newForceRemoveHarness( fmt.Sprintf("node-%d:7000", id), fmt.Sprintf("node-%d:8000", id), fmt.Appendf(nil, "%016d", id), + nil, )) } @@ -438,7 +439,7 @@ func TestForceRemoveNodeDurabilityFailureContract(t *testing.T) { require.ErrorIs(t, runErr, injected) _, err = h.n.GetClusterState(context.Background()) require.ErrorIs(t, err, injected) - _, err = h.n.Propose(context.Background(), NewProposal(100, []byte("must not be admitted"))) + _, err = h.n.Propose(context.Background(), newProposal(100, []byte("must not be admitted"), nil)) require.ErrorIs(t, err, injected) _, restartedVoters := h.restartedState(t) @@ -497,7 +498,7 @@ func TestForceRemoveNodeDurabilityFailureContract(t *testing.T) { synctest.Wait() require.Len(t, h.n.clusterCommandCh, 1) - proposal := NewProposal(101, []byte("queued before terminal publication")) + proposal := newProposal(101, []byte("queued before terminal publication"), nil) trackerBefore := h.n.indexTracker.Next() h.n.indexTracker.Lock() fsmFuture, err := h.n.Propose(context.Background(), proposal) diff --git a/internal/infra/node/force_remove_run_test.go b/internal/infra/node/force_remove_run_test.go index 2a5bc88d6a..4ec34cfca7 100644 --- a/internal/infra/node/force_remove_run_test.go +++ b/internal/infra/node/force_remove_run_test.go @@ -34,7 +34,7 @@ func TestForceRemoveNodeDurabilityFailureStopsRun(t *testing.T) { started: make(chan struct{}), } m := newTestMembership(t) - require.NoError(t, m.Register(2, "node-2:7000", "node-2:8000", []byte("0000000000000002"))) + require.NoError(t, m.Register(2, "node-2:7000", "node-2:8000", []byte("0000000000000002"), nil)) n, err := NewNode( NodeConfig{ NodeID: 1, diff --git a/internal/infra/node/historical_removal_lifecycle_test.go b/internal/infra/node/historical_removal_lifecycle_test.go index 1c581818d6..797d457d44 100644 --- a/internal/infra/node/historical_removal_lifecycle_test.go +++ b/internal/infra/node/historical_removal_lifecycle_test.go @@ -25,7 +25,7 @@ func newHistoricalRemovalLifecycleSetup(t *testing.T) (*Node, *testApplierSetup) }) var err error m, err = membership.NewMembership(membership.NewPeerStore(setup.store), noopMemTransport{}, noopMemPool{}, - 1, "self:7777", "self:8888", []byte("self-instance-id"), logging.Testing()) + 1, "self:7777", "self:8888", []byte("self-instance-id"), logging.Testing(), membership.NoopClusterPolicyProvider) require.NoError(t, err) n := &Node{fsm: setup.fsm, membership: m, logger: logging.Testing(), runDone: make(chan struct{})} t.Cleanup(func() { close(n.runDone) }) @@ -119,7 +119,7 @@ func TestHistoricalRemovalLifecycleDurableCleanupCannotClearLaterEvent(t *testin n, setup := newHistoricalRemovalLifecycleSetup(t) oldIdentity := []byte("old-instance-id-") newIdentity := []byte("new-instance-id-") - require.NoError(t, n.membership.Register(2, "peer:7777", "peer:8888", oldIdentity)) + require.NoError(t, n.membership.Register(2, "peer:7777", "peer:8888", oldIdentity, nil)) oldPending, err := n.trackCommittedRemoval(2, oldIdentity, 1) require.NoError(t, err) newPending, err := n.trackCommittedRemoval(2, newIdentity, 3) @@ -179,7 +179,7 @@ func TestNewNodeReplaysHistoricalRemovalCycles(t *testing.T) { fixture, setup := newHistoricalRemovalLifecycleSetup(t) oldIdentity := []byte("old-instance-id-") newIdentity := []byte("new-instance-id-") - require.NoError(t, fixture.membership.Register(2, "peer:7777", "peer:8888", oldIdentity)) + require.NoError(t, fixture.membership.Register(2, "peer:7777", "peer:8888", oldIdentity, nil)) // Retain the pre-crash ConfState: all committed changes exist only in the // WAL, so NewNode must recover their durable membership effects by replay. require.NoError(t, setup.wal.UpdateSnapshotConfState(&raftpb.ConfState{Voters: []uint64{1}, Learners: []uint64{2}})) diff --git a/internal/infra/node/historical_removal_test.go b/internal/infra/node/historical_removal_test.go index 395b5d3af7..001dbec650 100644 --- a/internal/infra/node/historical_removal_test.go +++ b/internal/infra/node/historical_removal_test.go @@ -37,11 +37,11 @@ func TestHistoricalRemovalFollowerReplay(t *testing.T) { }) var err error m, err = membership.NewMembership(membership.NewPeerStore(setup.store), noopMemTransport{}, noopMemPool{}, - 3, "self:7777", "self:8888", []byte("self-instance-id"), logging.Testing()) + 3, "self:7777", "self:8888", []byte("self-instance-id"), logging.Testing(), membership.NoopClusterPolicyProvider) require.NoError(t, err) oldIdentity := []byte("old-instance-id-") newIdentity := []byte("new-instance-id-") - require.NoError(t, m.Register(2, "old:7777", "old:8888", oldIdentity)) + require.NoError(t, m.Register(2, "old:7777", "old:8888", oldIdentity, nil)) confState := &raftpb.ConfState{Voters: []uint64{1, 3}, Learners: []uint64{2}} require.NoError(t, setup.wal.UpdateSnapshotConfState(confState)) diff --git a/internal/infra/node/learner_admission_test.go b/internal/infra/node/learner_admission_test.go index 66528b8038..f9ee97afc7 100644 --- a/internal/infra/node/learner_admission_test.go +++ b/internal/infra/node/learner_admission_test.go @@ -12,6 +12,7 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" "github.com/formancehq/ledger/v3/internal/infra/membership" + "github.com/formancehq/ledger/v3/internal/storage/dal" ) func TestAddLearnerRejectsRemovedAndActiveIncarnations(t *testing.T) { @@ -81,14 +82,16 @@ func TestAddLearnerZeroProgressRefreshProposesCompleteReplacement(t *testing.T) synctest.Test(t, func(t *testing.T) { n := newConfiguredPeersTestNode(t) n.logger = logging.Testing() - setup := newTestApplierSetupWithConfChangeHandler(t, make(LocalResponses, 1024), n.membership.WriteConfChange) + setup := newTestApplierSetupWithConfChangeHandler(t, make(LocalResponses, 1024), func(entry *raftpb.Entry, session *dal.WriteSession) error { + return n.membership.WriteConfChange(entry, session) + }) n.fsm = setup.fsm require.Zero(t, n.rawNode.Status().Progress[2].Match) ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() results := make(chan error, 1) go func() { - results <- n.AddLearner(ctx, 2, "new:7777", "new:8888", []byte("next-instance-id")) + results <- n.AddLearner(ctx, 2, "new:7777", "new:8888", []byte("next-instance-id"), nil) }() cmd := <-n.clusterCommandCh require.NoError(t, cmd.fn()) @@ -130,3 +133,51 @@ func TestAddLearnerZeroProgressRefreshProposesCompleteReplacement(t *testing.T) require.NoError(t, receiveForceRemoveError(t, results)) }) } + +func TestRefreshMembershipProposesUpdateForExistingIdentity(t *testing.T) { + t.Parallel() + + n := newConfiguredPeersTestNode(t) + n.logger = logging.Testing() + setup := newTestApplierSetupWithConfChangeHandler(t, make(LocalResponses, 1024), func(entry *raftpb.Entry, session *dal.WriteSession) error { + return n.membership.WriteConfChange(entry, session) + }) + n.fsm = setup.fsm + + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + results := make(chan error, 1) + go func() { + results <- n.RefreshMembership(ctx, 2, "refreshed:7777", "refreshed:8888", []byte("peer-instance-id"), []uint32{1, 2}) + }() + + cmd := <-n.clusterCommandCh + require.NoError(t, cmd.fn()) + ready := n.rawNode.Ready() + var changes []*raftpb.Entry + for _, entry := range ready.Entries { + if entry.GetType() == raftpb.EntryConfChangeV2 { + changes = append(changes, entry) + } + } + require.Len(t, changes, 1) + cc, ok, err := membership.UnmarshalConfChangeV2(changes[0]) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, raftpb.ConfChangeUpdateNode, cc.GetChanges()[0].GetType()) + payload, err := membership.UnmarshalConfChangeContext(cc.GetContext()) + require.NoError(t, err) + require.Equal(t, []uint32{1, 2}, payload.Capabilities) + require.Equal(t, []byte("peer-instance-id"), payload.InstanceID) + require.Equal(t, "refreshed:7777", payload.RaftAddress) + + pending, err := n.takePendingConfChange(cc, changes[0].GetIndex()) + require.NoError(t, err) + prefix, err := n.wal.Entries(1, changes[0].GetIndex(), ^uint64(0)) + require.NoError(t, err) + _, err = setup.fsm.ApplyEntries(t.Context(), setup.store, append(prefix, changes[0])...) + require.NoError(t, err) + pending.future.Resolve(pending.index, nil) + cmd.errCh <- nil + require.NoError(t, receiveForceRemoveError(t, results)) +} diff --git a/internal/infra/node/member_ids_test.go b/internal/infra/node/member_ids_test.go index 8c24e054c7..5fd59b96bb 100644 --- a/internal/infra/node/member_ids_test.go +++ b/internal/infra/node/member_ids_test.go @@ -5,6 +5,8 @@ import ( "github.com/stretchr/testify/require" "go.etcd.io/raft/v3/raftpb" + + "github.com/formancehq/ledger/v3/internal/infra/capabilities" ) func TestNodeMemberIDs(t *testing.T) { @@ -22,3 +24,21 @@ func TestNodeMemberIDs(t *testing.T) { require.Equal(t, []uint64{1, 2, 3, 4, 5}, n.MemberIDs()) } + +func TestCapabilityMemberRolesIncludesJointConfiguration(t *testing.T) { + t.Parallel() + + roles := capabilityMemberRoles(&raftpb.ConfState{ + Voters: []uint64{1}, + Learners: []uint64{4}, + VotersOutgoing: []uint64{2}, + LearnersNext: []uint64{5}, + }) + + require.Equal(t, map[uint64]capabilities.MemberRole{ + 1: capabilities.Voter, + 2: capabilities.Voter, + 4: capabilities.Learner, + 5: capabilities.Learner, + }, roles) +} diff --git a/internal/infra/node/membership_helpers_test.go b/internal/infra/node/membership_helpers_test.go index 21c7da255f..c350cbec96 100644 --- a/internal/infra/node/membership_helpers_test.go +++ b/internal/infra/node/membership_helpers_test.go @@ -47,6 +47,7 @@ func newTestMembership(t *testing.T) *membership.Membership { 42, "self:7777", "self:8888", []byte("self-instance-id"), logging.Testing(), + membership.NoopClusterPolicyProvider, ) require.NoError(t, err) m.Start() @@ -70,6 +71,7 @@ func TestRegisterInitialPeersPersistsInstanceIDs(t *testing.T) { Address: "peer:7777", ServiceAddress: "peer:8888", InstanceID: instanceID, + Capabilities: []uint32{7, 9}, }}, }, true) require.NoError(t, err) @@ -80,6 +82,7 @@ func TestRegisterInitialPeersPersistsInstanceIDs(t *testing.T) { storedSelf, ok := m.GetInstanceID(42) require.True(t, ok) require.Equal(t, selfInstanceID, storedSelf) + require.Equal(t, []uint32{7, 9}, m.PeerAddresses()[1].Capabilities) } func TestRegisterInitialPeersRejectsMissingInstanceID(t *testing.T) { diff --git a/internal/infra/node/membership_invariant_test.go b/internal/infra/node/membership_invariant_test.go index e950c01b59..bd670a72d5 100644 --- a/internal/infra/node/membership_invariant_test.go +++ b/internal/infra/node/membership_invariant_test.go @@ -16,7 +16,7 @@ func TestLeaderMembershipOperationsRejectMissingRowBeforeMutation(t *testing.T) for name, invoke := range map[string]func(context.Context, *Node) error{ "add learner": func(ctx context.Context, n *Node) error { - return n.AddLearner(ctx, 2, "peer:7777", "peer:8888", []byte("next-instance-id")) + return n.AddLearner(ctx, 2, "peer:7777", "peer:8888", []byte("next-instance-id"), nil) }, "remove node": func(ctx context.Context, n *Node) error { return n.RemoveNode(ctx, 2) diff --git a/internal/infra/node/membership_leader_ready_test.go b/internal/infra/node/membership_leader_ready_test.go index ff27d6d94a..6126dadd21 100644 --- a/internal/infra/node/membership_leader_ready_test.go +++ b/internal/infra/node/membership_leader_ready_test.go @@ -48,7 +48,7 @@ func TestAddLearnerWaitsForLeaderReady(t *testing.T) { results := make(chan error, 1) go func() { - results <- n.AddLearner(t.Context(), 2, "new:7777", "new:8888", []byte("next-instance-id")) + results <- n.AddLearner(t.Context(), 2, "new:7777", "new:8888", []byte("next-instance-id"), nil) }() // The command channel is unbuffered: an admission that reached it would diff --git a/internal/infra/node/node.go b/internal/infra/node/node.go index b19282acc6..bf6263bbc4 100644 --- a/internal/infra/node/node.go +++ b/internal/infra/node/node.go @@ -26,6 +26,7 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" "github.com/formancehq/ledger/v3/internal/domain" + "github.com/formancehq/ledger/v3/internal/infra/capabilities" "github.com/formancehq/ledger/v3/internal/infra/membership" "github.com/formancehq/ledger/v3/internal/infra/state" "github.com/formancehq/ledger/v3/internal/pkg/futures" @@ -398,6 +399,10 @@ type Node struct { drainOutstanding uint64 // guarded by proposalAdmissionMu drainReleaseSeen uint64 // guarded by proposalAdmissionMu drainGeneration uint64 // guarded by proposalAdmissionMu + // proposalAdmission is the authoritative policy gate immediately before a + // proposal enters the Raft queue. It is injected at construction so every + // proposal producer shares the same serialized-boundary check. + proposalAdmission ProposalAdmission // membership owns the Raft peer-address state (Pebble + in-memory // cache) and the OnSnapshotInstalled / WriteConfChange callbacks // wired into Applier and Machine. EN-1413. @@ -419,7 +424,7 @@ type Node struct { // RemoveNode, PromoteLearner) so that only one proposal is in-flight at a // time. This avoids unnecessary retries caused by etcd/raft silently // dropping concurrent ConfChange proposals. - confChangeMu sync.Mutex + confChangeMu membership.Coordinator // lastAutoPromote tracks the last time an auto-promotion was proposed for // each learner node. Used to rate-limit proposals and avoid spamming raft @@ -468,6 +473,18 @@ type Node struct { readIndexDurationHistogram metric.Float64Histogram } +// ProposalAdmission is the authoritative admission check applied immediately +// before a proposal is queued for Raft. +type ProposalAdmission func(context.Context, *Proposal) error + +// NodeOption configures a Node dependency at construction time. +type NodeOption func(*Node) + +// WithProposalAdmission installs the serialized-boundary proposal gate. +func WithProposalAdmission(admission ProposalAdmission) NodeOption { + return func(node *Node) { node.proposalAdmission = admission } +} + // NewNode creates a new wrapper around a RawNode. func NewNode( cfg NodeConfig, @@ -481,6 +498,7 @@ func NewNode( synchronizer *state.Synchronizer, membership *membership.Membership, localResponses LocalResponses, + opts ...NodeOption, ) (*Node, error) { if err := cfg.validateTickIntervals(); err != nil { return nil, err @@ -689,7 +707,7 @@ func NewNode( // writes self for Bootstrap/Restore, but the restart branch never // did — that's the gap the previous bootstrap hook used to cover // before transport wiring moved into Membership. - if err := membership.Register(cfg.NodeID, cfg.AdvertiseAddr, cfg.ServiceAdvertiseAddr, cfg.InstanceID); err != nil { + if err := membership.Register(cfg.NodeID, cfg.AdvertiseAddr, cfg.ServiceAdvertiseAddr, cfg.InstanceID, capabilities.IDsAsUint32(capabilities.SupportedIDs())); err != nil { return nil, fmt.Errorf("refreshing self in peer store: %w", err) } @@ -716,6 +734,10 @@ func NewNode( indexTracker: NewIndexTracker(initialIndex(wal)), observer: NewNoOpObserver(), } + for _, opt := range opts { + opt(node) + } + node.confChangeMu.Configure(node.waitForPriorMembership, node.stopChannel, node.terminalCh) // Start ready: no leadership transition is in progress yet. node.leaderReady.Store(newCompletedLeaderReadyGeneration()) @@ -1750,7 +1772,10 @@ func (node *Node) finishReady(result readyResult, stop chan struct{}) error { err := membership.WalkConfChangeContexts(cc, func(t raftpb.ConfChangeType, nodeID uint64, ctx *membership.ConfChangeContext) error { switch t { case raftpb.ConfChangeAddNode, raftpb.ConfChangeAddLearnerNode, raftpb.ConfChangeUpdateNode: - if err := node.membership.Set(nodeID, ctx.RaftAddress, ctx.ServiceAddress, ctx.InstanceID); err != nil { + if ctx == nil { + return fmt.Errorf("invariant: committed registration for node %d has no identity context", nodeID) + } + if err := node.membership.Set(nodeID, ctx.RaftAddress, ctx.ServiceAddress, ctx.InstanceID, ctx.Capabilities); err != nil { return fmt.Errorf("invariant: applying membership cache update: %w", err) } case raftpb.ConfChangeRemoveNode: @@ -1776,7 +1801,6 @@ func (node *Node) finishReady(result readyResult, stop chan struct{}) error { if err != nil { return err } - pending, err := node.takePendingConfChange(cc, committed.index) if err != nil { return err @@ -1794,7 +1818,6 @@ func (node *Node) finishReady(result readyResult, stop chan struct{}) error { }) } } - // If the ConfState changed (e.g. a learner was added), update the WAL // snapshot's ConfState immediately. Without this, etcd/raft would send // the stale snapshot (which lacks the new node) before the applier's @@ -2364,17 +2387,34 @@ func (node *Node) Propose(ctx context.Context, proposal *Proposal) (*futures.Fut if err := node.terminalError(); err != nil { return nil, err } - if released := node.fsm.DrainReleaseIndex(); released > node.drainReleaseSeen { - node.drainReleaseSeen = released - node.drainGeneration++ - node.drainOutstanding = 0 + if node.fsm != nil { + if released := node.fsm.DrainReleaseIndex(); released > node.drainReleaseSeen { + node.drainReleaseSeen = released + node.drainGeneration++ + node.drainOutstanding = 0 + } } if !proposal.drainMarker && !proposal.releaseMarker && !proposal.membershipBarrier { - if _, index := node.fsm.Drain(); index != 0 || node.drainOutstanding != 0 { + var drainIndex uint64 + if node.fsm != nil { + _, drainIndex = node.fsm.Drain() + } + if drainIndex != 0 || node.drainOutstanding != 0 { return nil, domain.ErrMaintenanceMode } } + // Evaluate the authoritative policy gate while proposalAdmissionMu and the + // IndexTracker guard are held by the caller. A rejected proposal therefore + // never consumes a predicted Raft index and can never leave a hole for the + // proposals already queued behind it. Policy-changing proposals use the + // same canonical runner, so this is also their serialization point. + if node.proposalAdmission != nil { + if err := node.proposalAdmission(ctx, proposal); err != nil { + return nil, err + } + } + // Create a separate future for Machine results. // The proposal's embedded Future is for Raft consensus (resolved by rawNode.Propose). // The fsmFuture is for Machine processing (resolved when entry is applied). @@ -2415,10 +2455,9 @@ func (node *Node) Propose(ctx context.Context, proposal *Proposal) (*futures.Fut } } -// rejectQueuedProposal resolves a proposal that was admitted before a terminal -// command failure but can no longer be handed to rawNode. The admission caller -// has released the tracker guard before this runs, so Decrement takes the -// regular synchronized rollback path. +// rejectQueuedProposal resolves a proposal drained during terminal shutdown. +// Normal capability rejection happens before index reservation in Propose and +// never reaches the queue. func (node *Node) rejectQueuedProposal(proposal *Proposal, err error) { node.indexTracker.Decrement(1) node.applier.ResolveDroppedFuture(proposal.commandID, err) @@ -2546,6 +2585,65 @@ func (node *Node) MemberIDs() []uint64 { return ids } +// CapabilityMembers returns a lock-consistent membership evidence snapshot for +// admission. Missing durable rows are represented as unreachable members so +// activation cannot proceed on an incomplete directory. +func (node *Node) CapabilityMembers() []capabilities.Member { + confState := node.confState.Load() + if confState == nil { + return nil + } + addresses := node.membership.PeerAddresses() + roles := capabilityMemberRoles(confState) + result := make([]capabilities.Member, 0, len(roles)) + for id, role := range roles { + peer, ok := addresses[id] + member := capabilities.Member{NodeID: id, Role: role, Reachable: ok && peer.RaftAddress != "" && peer.ServiceAddress != ""} + if ok { + member.InstanceID = peer.InstanceID + member.Capabilities = make([]capabilities.CapabilityID, len(peer.Capabilities)) + for i, capabilityID := range peer.Capabilities { + member.Capabilities[i] = capabilities.CapabilityID(capabilityID) + } + } + result = append(result, member) + } + slices.SortFunc(result, func(a, b capabilities.Member) int { + if a.NodeID < b.NodeID { + return -1 + } + if a.NodeID > b.NodeID { + return 1 + } + + return 0 + }) + + return result +} + +func capabilityMemberRoles(confState *raftpb.ConfState) map[uint64]capabilities.MemberRole { + roles := make(map[uint64]capabilities.MemberRole) + for _, ids := range [][]uint64{ + confState.GetVoters(), + confState.GetVotersOutgoing(), + } { + for _, id := range ids { + roles[id] = capabilities.Voter + } + } + for _, ids := range [][]uint64{ + confState.GetLearners(), + confState.GetLearnersNext(), + } { + for _, id := range ids { + roles[id] = capabilities.Learner + } + } + + return roles +} + // Logger returns the node's logger. func (node *Node) Logger() logging.Logger { return node.logger @@ -2624,6 +2722,7 @@ func (node *Node) GetConfiguredPeers(ctx context.Context) ([]Peer, error) { Address: address.RaftAddress, ServiceAddress: address.ServiceAddress, InstanceID: bytes.Clone(address.InstanceID), + Capabilities: append([]uint32(nil), address.Capabilities...), }) } @@ -3112,8 +3211,11 @@ func classifyExistingLearner(match uint64, existingInstanceID, incomingInstanceI // the explicit call path selects the admission semantics. // // Must be called on the leader. -func (node *Node) AddLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte) error { - return node.addLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, false) +// AddLearner proposes a learner registration carrying the caller's capability +// proof. The proof is persisted with the peer identity and is required by the +// active capability policy. +func (node *Node) AddLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { + return node.addLearnerWithCapabilities(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs, false) } // WithStableDrainMembership serializes a drain capability preflight and its @@ -3141,23 +3243,116 @@ func (node *Node) WithStableDrainMembership(ctx context.Context, fn func() error return fn() } +// MembershipCoordinator returns the shared guard for membership changes and +// capability policy activation. +func (node *Node) MembershipCoordinator() *membership.Coordinator { + return &node.confChangeMu +} + // JoinAsLearner is the bootstrap admission path for a process with an empty // WAL. It deliberately keeps EN-1436's stricter stale-progress handling -// separate from administrative AddLearner retries. -func (node *Node) JoinAsLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte) error { - return node.addLearner(ctx, nodeID, raftAddr, serviceAddr, instanceID, true) +// separate from administrative AddLearner retries and carries the joining +// node's capability evidence. +func (node *Node) JoinAsLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { + return node.addLearnerWithCapabilities(ctx, nodeID, raftAddr, serviceAddr, instanceID, capabilityIDs, true) +} + +// RefreshMembership proposes a committed UpdateNode for an existing member. +// Unlike JoinAsLearner, this path is used by a node with an already populated +// WAL after a normal restart. It requires the same persisted instance identity +// and replaces the committed endpoint/capability evidence through Raft. +// Must be called on the leader. +func (node *Node) RefreshMembership(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32) error { + if err := membership.ValidateInstanceID(instanceID); err != nil { + return fmt.Errorf("invalid member identity: %w", err) + } + proposalID := uuid.NewString() + ccCtx, err := membership.MarshalConfChangeContext(membership.ConfChangeContext{ + RaftAddress: raftAddr, + ServiceAddress: serviceAddr, + InstanceID: instanceID, + Capabilities: append([]uint32(nil), capabilityIDs...), + ProposalID: proposalID, + }) + if err != nil { + return fmt.Errorf("marshaling membership refresh context: %w", err) + } + + return node.retryConfChange(ctx, proposalID, nodeID, "RefreshMembership", []raftpb.ConfChangeType{ + raftpb.ConfChangeUpdateNode, + }, func() error { + status := node.rawNode.Status() + if status.RaftState != raft.StateLeader { + return ErrNotLeader + } + + if _, ok := status.Progress[nodeID]; !ok { + return ErrNodeNotInCluster + } + + existing, ok := node.membership.GetInstanceID(nodeID) + if !ok { + return fmt.Errorf("member %d has no persisted identity", nodeID) + } + if err := membership.ValidateInstanceID(existing); err != nil { + return fmt.Errorf("member %d has invalid persisted identity: %w", nodeID, err) + } + if !bytes.Equal(existing, instanceID) { + return fmt.Errorf("member %d identity does not match the active incarnation", nodeID) + } + + if err := node.membership.ValidateTransitionEvidence(&membership.ConfChangeContext{ + RaftAddress: raftAddr, + ServiceAddress: serviceAddr, + InstanceID: instanceID, + Capabilities: capabilityIDs, + }); err != nil { + return err + } + + if node.isRemovalPending(nodeID, instanceID) { + return ErrNodeRemoved + } + removed, checkErr := node.membership.IsRemoved(nodeID, instanceID) + if checkErr != nil { + return fmt.Errorf("checking removed-member registry for %d: %w", nodeID, checkErr) + } + if removed { + return ErrNodeRemoved + } + + return node.rawNode.ProposeConfChange(&raftpb.ConfChangeV2{ + Changes: []*raftpb.ConfChangeSingle{{ + Type: new(raftpb.ConfChangeUpdateNode), + NodeId: new(nodeID), + }}, + Context: ccCtx, + }) + }, nil) } +// SetCapabilityGate updates the leader-side membership admission gate from a +// committed ClusterPolicy projection. +func (node *Node) SetCapabilityGate(required []uint32) error { + return node.membership.SetCapabilityGate(required) +} + +// addLearner is kept as a compatibility shim for package-local tests and +// callers that exercise the pre-evidence admission path directly. func (node *Node) addLearner(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, bootJoin bool) error { + return node.addLearnerWithCapabilities(ctx, nodeID, raftAddr, serviceAddr, instanceID, nil, bootJoin) +} + +func (node *Node) addLearnerWithCapabilities(ctx context.Context, nodeID uint64, raftAddr, serviceAddr string, instanceID []byte, capabilityIDs []uint32, bootJoin bool) error { if err := membership.ValidateInstanceID(instanceID); err != nil { return fmt.Errorf("invalid learner identity: %w", err) } - proposalID := uuid.NewString() ccCtx, err := membership.MarshalConfChangeContext(membership.ConfChangeContext{ RaftAddress: raftAddr, ServiceAddress: serviceAddr, InstanceID: instanceID, + Capabilities: append([]uint32(nil), capabilityIDs...), ProposalID: proposalID, }) if err != nil { @@ -3172,6 +3367,17 @@ func (node *Node) addLearner(ctx context.Context, nodeID uint64, raftAddr, servi if status.RaftState != raft.StateLeader { return ErrNotLeader } + // Revalidate immediately before RawNode.Propose: policy activation may + // have changed while the request waited for the serialized ConfChange + // boundary. + if err := node.membership.ValidateTransitionEvidence(&membership.ConfChangeContext{ + RaftAddress: raftAddr, + ServiceAddress: serviceAddr, + InstanceID: instanceID, + Capabilities: capabilityIDs, + }); err != nil { + return err + } // EN-1045 defense against a race between JoinAsLearner admission // (Pebble IsRemoved check) and RemoveNode commit: re-check the @@ -3266,6 +3472,13 @@ func (node *Node) PromoteLearner(ctx context.Context, nodeID uint64) error { if err != nil { return err } + promotionCtx, decodeErr := membership.UnmarshalConfChangeContext(ccCtx) + if decodeErr != nil { + return fmt.Errorf("decoding learner membership evidence: %w", decodeErr) + } + if err := node.membership.ValidateTransitionEvidence(&promotionCtx); err != nil { + return err + } return node.rawNode.ProposeConfChange(&raftpb.ConfChangeV2{ Changes: []*raftpb.ConfChangeSingle{{ @@ -3846,7 +4059,7 @@ func initialJoinVoters(peers []Peer, selfID uint64) []uint64 { // through WriteConfChange). func registerInitialPeers(m *membership.Membership, cfg NodeConfig, includeSelf bool) error { if includeSelf { - if err := m.Register(cfg.NodeID, cfg.AdvertiseAddr, cfg.ServiceAdvertiseAddr, cfg.InstanceID); err != nil { + if err := m.Register(cfg.NodeID, cfg.AdvertiseAddr, cfg.ServiceAdvertiseAddr, cfg.InstanceID, capabilities.IDsAsUint32(capabilities.SupportedIDs())); err != nil { return fmt.Errorf("persisting self in peer store: %w", err) } } @@ -3854,7 +4067,7 @@ func registerInitialPeers(m *membership.Membership, cfg NodeConfig, includeSelf // Peer discovery carries the durable identity of every configured member; // never create an address-only membership row. for _, p := range cfg.Peers { - if err := m.Register(p.ID, p.Address, p.ServiceAddress, p.InstanceID); err != nil { + if err := m.Register(p.ID, p.Address, p.ServiceAddress, p.InstanceID, p.Capabilities); err != nil { return fmt.Errorf("persisting initial peer %d: %w", p.ID, err) } } diff --git a/internal/infra/node/promotion_identity_test.go b/internal/infra/node/promotion_identity_test.go index 72f322285f..233cbc9a4b 100644 --- a/internal/infra/node/promotion_identity_test.go +++ b/internal/infra/node/promotion_identity_test.go @@ -14,6 +14,7 @@ import ( logging "github.com/formancehq/go-libs/v5/pkg/observe/log" "github.com/formancehq/ledger/v3/internal/infra/membership" + "github.com/formancehq/ledger/v3/internal/storage/dal" ) // requireCompletePromotion inspects the actual entry emitted by RawNode, so @@ -48,7 +49,9 @@ func TestPromoteLearner_ProposesCompleteRegisteredIdentity(t *testing.T) { t.Parallel() n := newConfiguredPeersTestNode(t) n.logger = logging.Testing() - setup := newTestApplierSetupWithConfChangeHandler(t, make(LocalResponses, 1024), n.membership.WriteConfChange) + setup := newTestApplierSetupWithConfChangeHandler(t, make(LocalResponses, 1024), func(entry *raftpb.Entry, session *dal.WriteSession) error { + return n.membership.WriteConfChange(entry, session) + }) n.fsm = setup.fsm results := make(chan error, 1) go func() { @@ -108,7 +111,7 @@ func TestCheckAndPromoteLearnersMissingRowStopsRun(t *testing.T) { Voters: []uint64{1}, Learners: []uint64{2}, })) m := newTestMembership(t) - require.NoError(t, m.Register(2, "node-2:7000", "node-2:8000", []byte("0000000000000002"))) + require.NoError(t, m.Register(2, "node-2:7000", "node-2:8000", []byte("0000000000000002"), nil)) n, err := NewNode( NodeConfig{ NodeID: 1, diff --git a/internal/infra/node/proposal.go b/internal/infra/node/proposal.go index 96f913fce7..1c47babe5a 100644 --- a/internal/infra/node/proposal.go +++ b/internal/infra/node/proposal.go @@ -1,6 +1,22 @@ package node -import "github.com/formancehq/ledger/v3/internal/pkg/futures" +import ( + "google.golang.org/protobuf/encoding/protowire" + + "github.com/formancehq/ledger/v3/internal/infra/capabilities" + "github.com/formancehq/ledger/v3/internal/pkg/futures" + "github.com/formancehq/ledger/v3/internal/pkg/protowireutil" + "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" +) + +var predictedIndexField = func() protowire.Number { + fd := (&raftcmdpb.Proposal{}).ProtoReflect().Descriptor().Fields().ByName("predicted_index") + if fd == nil { + panic("invariant: Proposal has no predicted_index field") + } + + return fd.Number() +}() type Proposal struct { *futures.Future[any] @@ -11,18 +27,99 @@ type Proposal struct { releaseMarker bool membershipBarrier bool // expectedTerm fences a drain preflight against a later leadership term. - expectedTerm uint64 + expectedTerm uint64 + requiredCapabilities []capabilities.CapabilityID + // activePolicyCapabilities is populated by the proposal factory for + // SetClusterPolicy proposals and orders. Keeping the typed candidate alongside the + // serialized command lets the leader validate activation evidence before + // handing the bytes to Raft without decoding the command a second time. + activePolicyCapabilities []capabilities.CapabilityID } -func NewProposal(commandID uint64, data []byte) *Proposal { +func newProposal(commandID uint64, data []byte, required []capabilities.CapabilityID) *Proposal { + // Reserve the wire tag and fixed64 value before the index tracker is held. + // SetPredictedIndex must only append into this buffer in its critical section. + buffer := make([]byte, len(data), len(data)+protowire.SizeTag(predictedIndexField)+8) + copy(buffer, data) + return &Proposal{ - commandID: commandID, - data: data, - Future: futures.New[any](), + commandID: commandID, + data: buffer, + requiredCapabilities: append([]capabilities.CapabilityID(nil), required...), + Future: futures.New[any](), } } +// NewProposal is the only public proposal factory. It derives the capability +// declaration and serializes the same final command, making it impossible for +// the typed admission metadata to diverge from the wire payload. +func NewProposal(commandID uint64, command *raftcmdpb.Proposal) (*Proposal, error) { + prepared, err := capabilities.PrepareProposal(command) + if err != nil { + return nil, err + } + + data, err := command.MarshalVT() + if err != nil { + return nil, err + } + + proposal := newProposal(commandID, data, prepared.RequiredCapabilities()) + // Validate the complete serialized command before it enters the Raft queue. + // SetPredictedIndex only appends a fixed-width index field afterward; it + // cannot change the payload or its capability declaration. + if err := capabilities.ValidateFinalProposal(data, proposal.requiredCapabilities); err != nil { + return nil, err + } + if policy := command.GetSetClusterPolicy().GetPolicy(); policy != nil { + for _, activation := range policy.GetCapabilityActivations() { + if activation != nil && activation.GetActive() { + proposal.activePolicyCapabilities = append(proposal.activePolicyCapabilities, capabilities.CapabilityID(activation.GetId())) + } + } + } + for _, order := range command.GetOrders() { + policy := order.GetSystemScoped().GetSetClusterPolicy().GetPolicy() + if policy == nil { + continue + } + for _, activation := range policy.GetCapabilityActivations() { + if activation != nil && activation.GetActive() { + proposal.activePolicyCapabilities = append(proposal.activePolicyCapabilities, capabilities.CapabilityID(activation.GetId())) + } + } + } + + return proposal, nil +} + // Data returns the serialized proposal data. func (p *Proposal) Data() []byte { - return p.data + return append([]byte(nil), p.data...) +} + +// SetPredictedIndex patches the predicted Raft index onto the already +// serialized proposal. NewProposal deliberately serializes with a zero index; +// callers can therefore perform the expensive payload serialization before +// acquiring the shared proposal lock and hold it only for this bounded append. +func (p *Proposal) SetPredictedIndex(index uint64) { + if index == 0 { + return + } + + p.data = protowireutil.AppendFixed64(p.data, predictedIndexField, index) +} + +// RequiredCapabilities returns the capabilities derived by the producer from +// the final command before it was serialized. Admission reuses this typed +// value instead of decoding the serialized proposal a second time. +func (p *Proposal) RequiredCapabilities() []capabilities.CapabilityID { + return append([]capabilities.CapabilityID(nil), p.requiredCapabilities...) +} + +// ActivePolicyCapabilities returns the active capabilities requested by a +// SetClusterPolicy proposal or order, if any. The values are derived by NewProposal from +// the typed command and are not read from the serialized payload. +func (p *Proposal) ActivePolicyCapabilities() []capabilities.CapabilityID { + return append([]capabilities.CapabilityID(nil), p.activePolicyCapabilities...) } diff --git a/internal/infra/node/proposal_admission_test.go b/internal/infra/node/proposal_admission_test.go new file mode 100644 index 0000000000..db82de066f --- /dev/null +++ b/internal/infra/node/proposal_admission_test.go @@ -0,0 +1,25 @@ +package node + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestProposeAppliesAdmissionBeforeIndexReservation(t *testing.T) { + rejected := errors.New("capability policy changed before Raft proposal") + + n := &Node{ + indexTracker: NewIndexTracker(10), + proposalAdmission: func(context.Context, *Proposal) error { + return rejected + }, + } + + proposal := newProposal(42, []byte("proposal"), nil) + _, err := n.Propose(context.Background(), proposal) + require.ErrorIs(t, err, rejected) + require.Equal(t, uint64(10), n.indexTracker.Next()) +} diff --git a/internal/infra/node/proposal_test.go b/internal/infra/node/proposal_test.go new file mode 100644 index 0000000000..2f93522166 --- /dev/null +++ b/internal/infra/node/proposal_test.go @@ -0,0 +1,107 @@ +package node + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + + "github.com/formancehq/ledger/v3/internal/infra/capabilities" + "github.com/formancehq/ledger/v3/internal/proto/commonpb" + "github.com/formancehq/ledger/v3/internal/proto/raftcmdpb" +) + +func TestNewProposalBindsCapabilitiesToSerializedCommand(t *testing.T) { + command := &raftcmdpb.Proposal{Id: 1} + proposal, err := NewProposal(1, command) + require.NoError(t, err) + + command.RequiredCapabilities = []uint32{99} + require.NotEqual(t, command.GetRequiredCapabilities(), proposal.RequiredCapabilities()) + + data := proposal.Data() + data[0] ^= 0xff + require.NotEqual(t, data, proposal.Data()) +} + +func TestProposalSetPredictedIndexPatchesSerializedPayload(t *testing.T) { + command := &raftcmdpb.Proposal{Id: 1} + proposal, err := NewProposal(1, command) + require.NoError(t, err) + + proposal.SetPredictedIndex(42) + + decoded := new(raftcmdpb.Proposal) + require.NoError(t, decoded.UnmarshalVT(proposal.Data())) + require.Equal(t, uint64(42), decoded.GetPredictedIndex()) +} + +func TestNewProposalAppendsPredictedIndexWithoutMutatingPrefix(t *testing.T) { + command := &raftcmdpb.Proposal{Id: 1} + proposal, err := NewProposal(1, command) + require.NoError(t, err) + + before := proposal.Data() + proposal.SetPredictedIndex(42) + after := proposal.Data() + require.Greater(t, len(after), len(before)) + require.Equal(t, before, after[:len(before)]) +} + +func TestNewProposalReservesPredictedIndexCapacity(t *testing.T) { + for _, size := range []int{77, 8192, 1048576} { + t.Run(fmt.Sprintf("size-%d", size), func(t *testing.T) { + command := &raftcmdpb.Proposal{Id: 1} + command.ProtoReflect().SetUnknown(protowire.AppendBytes(protowire.AppendTag(nil, 2047, protowire.BytesType), make([]byte, size))) + proposal, err := NewProposal(1, command) + require.NoError(t, err) + before := &proposal.data[0] + require.GreaterOrEqual(t, cap(proposal.data)-len(proposal.data), protowire.SizeTag(predictedIndexField)+8) + proposal.SetPredictedIndex(42) + require.Same(t, before, &proposal.data[0]) + }) + } +} + +func TestNewProposalDerivesActivePolicyCapabilities(t *testing.T) { + command := &raftcmdpb.Proposal{ + Id: 1, + Orders: []*raftcmdpb.Order{{ + Type: &raftcmdpb.Order_SystemScoped{SystemScoped: &raftcmdpb.SystemScopedOrder{ + Payload: &raftcmdpb.SystemScopedOrder_SetClusterPolicy{SetClusterPolicy: &raftcmdpb.SetClusterPolicyOrder{ + Policy: &commonpb.ClusterPolicy{CapabilityActivations: []*commonpb.CapabilityActivation{ + {Id: 7, Active: true}, + {Id: 8, Active: false}, + }}, + }}, + }}, + }}, + } + + proposal, err := NewProposal(1, command) + require.NoError(t, err) + require.Equal(t, []capabilities.CapabilityID{7}, proposal.ActivePolicyCapabilities()) + + active := proposal.ActivePolicyCapabilities() + active[0] = 99 + require.Equal(t, []capabilities.CapabilityID{7}, proposal.ActivePolicyCapabilities(), "candidate metadata must be defensive") +} + +func TestNewProposalDerivesActivePolicyCapabilitiesFromNativePolicy(t *testing.T) { + command := &raftcmdpb.Proposal{ + Id: 1, + Payload: &raftcmdpb.Proposal_SetClusterPolicy{ + SetClusterPolicy: &raftcmdpb.SetClusterPolicyProposal{ + Policy: &commonpb.ClusterPolicy{CapabilityActivations: []*commonpb.CapabilityActivation{ + {Id: 7, Active: true}, + {Id: 8, Active: false}, + }}, + }, + }, + } + + proposal, err := NewProposal(1, command) + require.NoError(t, err) + require.Equal(t, []capabilities.CapabilityID{7}, proposal.ActivePolicyCapabilities()) +} diff --git a/internal/infra/node/startup_membership_removal_test.go b/internal/infra/node/startup_membership_removal_test.go index ea7cb1ceed..6deee68e53 100644 --- a/internal/infra/node/startup_membership_removal_test.go +++ b/internal/infra/node/startup_membership_removal_test.go @@ -26,10 +26,10 @@ func TestNewNodeAcceptsReplayedRemovalBeforeConfStatePersistence(t *testing.T) { }) var err error m, err = membership.NewMembership(membership.NewPeerStore(setup.store), noopMemTransport{}, noopMemPool{}, - 1, "self:7777", "self:8888", []byte("self-instance-id"), logging.Testing()) + 1, "self:7777", "self:8888", []byte("self-instance-id"), logging.Testing(), membership.NoopClusterPolicyProvider) require.NoError(t, err) identity := []byte("peer-instance-id") - require.NoError(t, m.Register(2, "peer:7777", "peer:8888", identity)) + require.NoError(t, m.Register(2, "peer:7777", "peer:8888", identity, nil)) require.NoError(t, setup.wal.UpdateSnapshotConfState(&raftpb.ConfState{Voters: []uint64{1, 2}})) context, err := membership.MarshalConfChangeContext(membership.ConfChangeContext{InstanceID: identity}) require.NoError(t, err) diff --git a/internal/infra/plan/proposal_composition_test.go b/internal/infra/plan/proposal_composition_test.go index f0305cc6fe..9181c27471 100644 --- a/internal/infra/plan/proposal_composition_test.go +++ b/internal/infra/plan/proposal_composition_test.go @@ -46,10 +46,10 @@ func TestRun_RejectsUnsupportedTechnicalComposition(t *testing.T) { } proposer := &recordingProposer{tracker: tracker} marshalled := false - result, err := builder.Run(t.Context(), cmd, build, func(p *raftcmdpb.Proposal) ([]byte, error) { + result, err := builder.Run(t.Context(), cmd, build, func(p *raftcmdpb.Proposal) error { marshalled = true - return p.MarshalVT() + return nil }, proposer) require.ErrorIs(t, err, ErrUnsupportedTechnicalComposition) require.Nil(t, result) @@ -75,10 +75,10 @@ func TestRun_RejectsUnsupportedNativeTechnicalComposition(t *testing.T) { cmd := &raftcmdpb.Proposal{Payload: &raftcmdpb.Proposal_Batch{Batch: batch}} proposer := &recordingProposer{tracker: tracker} marshalled := false - result, err := builder.Run(t.Context(), cmd, build, func(p *raftcmdpb.Proposal) ([]byte, error) { + result, err := builder.Run(t.Context(), cmd, build, func(p *raftcmdpb.Proposal) error { marshalled = true - return p.MarshalVT() + return nil }, proposer) require.ErrorIs(t, err, ErrUnsupportedTechnicalComposition) require.Nil(t, result) @@ -104,7 +104,7 @@ func TestRun_AllowsSupportedTechnicalComposition(t *testing.T) { cmd.Orders = []*raftcmdpb.Order{{}} } proposeErr := errors.New("reached proposer") - _, err = builder.Run(t.Context(), cmd, build, (*raftcmdpb.Proposal).MarshalVT, stubFailingProposer{err: proposeErr}) + _, err = builder.Run(t.Context(), cmd, build, func(*raftcmdpb.Proposal) error { return nil }, stubFailingProposer{err: proposeErr}) require.ErrorIs(t, err, proposeErr) }) } @@ -129,7 +129,7 @@ func TestRun_AllowsStandaloneOperationalUpdate(t *testing.T) { require.NoError(t, err) cmd := &raftcmdpb.Proposal{TechnicalUpdates: []*raftcmdpb.TechnicalUpdate{update}} proposeErr := errors.New("reached proposer") - _, err = builder.Run(t.Context(), cmd, build, (*raftcmdpb.Proposal).MarshalVT, stubFailingProposer{err: proposeErr}) + _, err = builder.Run(t.Context(), cmd, build, func(*raftcmdpb.Proposal) error { return nil }, stubFailingProposer{err: proposeErr}) require.ErrorIs(t, err, proposeErr) }) } diff --git a/internal/infra/plan/runner.go b/internal/infra/plan/runner.go index 2f7c72c141..29b45864fc 100644 --- a/internal/infra/plan/runner.go +++ b/internal/infra/plan/runner.go @@ -95,27 +95,29 @@ type RunResult struct { // The caller is responsible for: // - Calling Build with the slice of WriteOperations the proposal // carries (emitting whatever metrics it likes around that call). -// - Marshaling the command via the marshalFn callback (proto marshal -// plus caller-specific instrumentation like commandSize.Record). +// - Finalizing the command via the finalizeFn callback (caller-specific +// instrumentation like commandSize.Record). // - Releasing the loaders via guard.ReleaseLoaders() at the // appropriate time. // // Run assigns coverage_bits (and production_bits when a productions // assigner was attached via WithProductions) on each operation right -// before every marshalFn invocation — both on the happy path and on the -// rare rebuild path under the proposal guard. The caller's marshalFn +// before every finalizeFn invocation — both on the happy path and on the +// rare rebuild path under the proposal guard. The caller's finalizeFn // must not touch those fields. // // On the happy path Run: // 1. attaches build.ExecutionPlan to cmd, // 2. iterates build.operations to assign coverage bits over plans, // 3. invokes the productions assigner if any, -// 4. calls marshalFn to serialize cmd, -// 5. acquires the proposal guard (IndexTracker mutex), -// 6. patches cmd.PredictedIndex from the tracker's next index, -// 7. re-runs steps 2-4 + marshal if the next-index generation shifted, -// 8. submits the proposal via proposer.Propose, -// 9. releases the proposal-lock side of the guard. +// 4. finalizes cmd before proposal construction, +// 5. marshals the proposal before acquiring the proposal guard, +// 6. acquires the proposal guard (IndexTracker mutex), +// 7. patches PredictedIndex from the tracker's next index, +// 8. re-runs steps 2-4 + marshal if the next-index generation shifted, +// 9. submits the proposal via proposer.Propose, +// +// 10. releases the proposal-lock side of the guard. // // PredictedIndex is always set. It is a no-cost backstop: the FSM // rejects with ErrStaleProposal if the actual Raft index doesn't @@ -125,7 +127,7 @@ func (p *Builder) Run( ctx context.Context, cmd *raftcmdpb.Proposal, build *BuildResult, - marshalFn func(*raftcmdpb.Proposal) ([]byte, error), + finalizeFn func(*raftcmdpb.Proposal) error, proposer Proposer, ) (*RunResult, error) { // Operational updates write directly to the shared apply batch. Only @@ -141,7 +143,7 @@ func (p *Builder) Run( } // Guarantee a unique command ID. The applier keys FSM futures by - // `cmd.Id` (`node.NewProposal(cmd.GetId(), data)` below); two in-flight + // `cmd.Id` (used by the node proposal factory); two in-flight // proposals sharing Id=0 would overwrite each other in // `Applier.StoreFuture`, hanging one waiter and routing the apply // result to the wrong caller. Several technical callers build @@ -168,7 +170,7 @@ func (p *Builder) Run( // cache reset between Build and apply would spuriously reject the // proposal as stale even though it never consulted the cache. if build.aggregate.AttributeKeysCount() == 0 { - return p.runWithoutPreload(ctx, cmd, build, marshalFn, proposer) + return p.runWithoutPreload(ctx, cmd, build, finalizeFn, proposer) } cmd.ExecutionPlan = build.ExecutionPlan @@ -178,7 +180,17 @@ func (p *Builder) Run( return nil, fmt.Errorf("%w: %w", ErrIncompleteCoverage, err) } - data, err := marshalFn(cmd) + if err := finalizeFn(cmd); err != nil { + build.ReleaseLoaders() + + return nil, fmt.Errorf("%w: %w", ErrMarshalProposal, err) + } + + // Serialize the full payload before taking the shared proposal lock. The + // predicted index is zero at this point and is patched below once the + // tracker guard is held. + cmd.PredictedIndex = 0 + proposal, err := node.NewProposal(cmd.GetId(), cmd) if err != nil { build.ReleaseLoaders() @@ -201,8 +213,6 @@ func (p *Builder) Run( return nil, fmt.Errorf("%w: %w", ErrAcquireProposalGuard, err) } - cmd.PredictedIndex = p.TrackerNext() - result := &RunResult{Guard: guard} if updatedPreloads != nil { @@ -219,20 +229,21 @@ func (p *Builder) Run( return nil, fmt.Errorf("%w: %w", ErrIncompleteCoverage, err) } - data, err = marshalFn(cmd) + if err := finalizeFn(cmd); err != nil { + guard.ReleaseAll() + + return nil, fmt.Errorf("%w (under guard): %w", ErrMarshalProposal, err) + } + + proposal, err = node.NewProposal(cmd.GetId(), cmd) if err != nil { guard.ReleaseAll() return nil, fmt.Errorf("%w (under guard): %w", ErrMarshalProposal, err) } - } else { - // Common path: patch the PredictedIndex onto the pre-marshaled - // buffer rather than re-marshaling the whole proposal (which - // can be megabytes for large batches). - data = AppendProposalPredictedIndex(data, cmd.GetPredictedIndex()) } - proposal := node.NewProposal(cmd.GetId(), data) + proposal.SetPredictedIndex(p.TrackerNext()) result.ProposeStartTime = time.Now() fsmFuture, err := proposer.Propose(ctx, proposal) @@ -309,7 +320,7 @@ func (p *Builder) runWithoutPreload( ctx context.Context, cmd *raftcmdpb.Proposal, build *BuildResult, - marshalFn func(*raftcmdpb.Proposal) ([]byte, error), + finalizeFn func(*raftcmdpb.Proposal) error, proposer Proposer, ) (*RunResult, error) { // The empty-needs path reads nothing from the cache, so neither @@ -335,14 +346,22 @@ func (p *Builder) runWithoutPreload( } else { cmd.ExecutionPlan = nil } - if err := build.applyBits(cmd, nil); err != nil { build.ReleaseLoaders() return nil, fmt.Errorf("%w: %w", ErrIncompleteCoverage, err) } - data, err := marshalFn(cmd) + if err := finalizeFn(cmd); err != nil { + build.ReleaseLoaders() + + return nil, fmt.Errorf("%w: %w", ErrMarshalProposal, err) + } + + // Marshal before taking the tracker lock; only the bounded predicted-index + // append below belongs in the critical section. + cmd.PredictedIndex = 0 + proposal, err := node.NewProposal(cmd.GetId(), cmd) if err != nil { build.ReleaseLoaders() @@ -355,10 +374,7 @@ func (p *Builder) runWithoutPreload( p.LockTracker() - cmd.PredictedIndex = p.TrackerNext() - data = AppendProposalPredictedIndex(data, cmd.GetPredictedIndex()) - - proposal := node.NewProposal(cmd.GetId(), data) + proposal.SetPredictedIndex(p.TrackerNext()) proposeStart := time.Now() fsmFuture, err := proposer.Propose(ctx, proposal) diff --git a/internal/infra/plan/runner_test.go b/internal/infra/plan/runner_test.go index 00025889a2..d3e15e306a 100644 --- a/internal/infra/plan/runner_test.go +++ b/internal/infra/plan/runner_test.go @@ -50,18 +50,18 @@ func TestRunWithoutPreload_ClearsPreSetExecutionPlan(t *testing.T) { } marshalled := false - marshalFn := func(p *raftcmdpb.Proposal) ([]byte, error) { + finalizeFn := func(p *raftcmdpb.Proposal) error { marshalled = true require.Nil(t, p.GetExecutionPlan(), "ExecutionPlan must be cleared before marshal") - return []byte{0}, nil + return nil } proposer := stubFailingProposer{err: errors.New("stub")} - _, err := builder.Run(context.Background(), cmd, build, marshalFn, proposer) + _, err := builder.Run(context.Background(), cmd, build, finalizeFn, proposer) require.Error(t, err, "stub proposer should bubble its error") - require.True(t, marshalled, "marshalFn must have run") + require.True(t, marshalled, "finalizeFn must have run") require.Nil(t, cmd.GetExecutionPlan(), "cmd.ExecutionPlan must be nil after Run on the no-preload fast path") } @@ -103,7 +103,7 @@ func TestRun_IdempotencyOnlyNeedsTakesFastPath(t *testing.T) { token: &preload.CleanupToken{}, } - marshalFn := func(p *raftcmdpb.Proposal) ([]byte, error) { + finalizeFn := func(p *raftcmdpb.Proposal) error { ep := p.GetExecutionPlan() require.NotNil(t, ep, "ExecutionPlan must survive to carry IdempotencyKeys") require.Empty(t, ep.GetAttributes(), "Attributes must be dropped on the fast path") @@ -111,12 +111,12 @@ func TestRun_IdempotencyOnlyNeedsTakesFastPath(t *testing.T) { require.Len(t, ep.GetIdempotencyKeys(), 1, "IdempotencyKeys must survive") require.Equal(t, "idem-only", ep.GetIdempotencyKeys()[0].GetKey()) - return []byte{0}, nil + return nil } proposer := stubFailingProposer{err: errors.New("stub")} - _, err := builder.Run(context.Background(), cmd, build, marshalFn, proposer) + _, err := builder.Run(context.Background(), cmd, build, finalizeFn, proposer) require.Error(t, err, "stub proposer should bubble its error") ep := cmd.GetExecutionPlan() diff --git a/internal/infra/plan/submit_technical.go b/internal/infra/plan/submit_technical.go index c1e214b311..ba50fee3a7 100644 --- a/internal/infra/plan/submit_technical.go +++ b/internal/infra/plan/submit_technical.go @@ -96,7 +96,7 @@ func submitTechnicalOnce(ctx context.Context, builder *Builder, proposer Propose result, err := builder.Run( ctx, cmd, build, - func(c *raftcmdpb.Proposal) ([]byte, error) { return c.MarshalVT() }, + func(*raftcmdpb.Proposal) error { return nil }, proposer, ) if err != nil { diff --git a/internal/infra/state/audit_replay.go b/internal/infra/state/audit_replay.go index 55b07f45ff..955c6fa69e 100644 --- a/internal/infra/state/audit_replay.go +++ b/internal/infra/state/audit_replay.go @@ -96,7 +96,7 @@ func NewAuditReplayer(logger logging.Logger, auditKey string) (*AuditReplayer, e nil, auditKey, 0, - func(_ *raftpb.Entry, _ *dal.WriteSession) error { return nil }, + func(_ *raftpb.Entry, _ *dal.WriteSession) error { return nil }, NoopCapabilityPolicyCommitHandler, ) if err != nil { _ = store.Close() diff --git a/internal/infra/state/checkpoint_pipeline_regression_test.go b/internal/infra/state/checkpoint_pipeline_regression_test.go index 15c2e3b5e2..824d08b73c 100644 --- a/internal/infra/state/checkpoint_pipeline_regression_test.go +++ b/internal/infra/state/checkpoint_pipeline_regression_test.go @@ -58,6 +58,7 @@ func TestPipelinedApplyWithCheckpointDoesNotDiverge(t *testing.T) { "test-cluster", 0, noopConfChangeHandler, + NoopCapabilityPolicyCommitHandler, ) require.NoError(t, err) require.NoError(t, NewRecovery(machine, dataStore).RecoverState()) diff --git a/internal/infra/state/cluster_policy_recovery_test.go b/internal/infra/state/cluster_policy_recovery_test.go index f85dd11499..b712c553a9 100644 --- a/internal/infra/state/cluster_policy_recovery_test.go +++ b/internal/infra/state/cluster_policy_recovery_test.go @@ -32,6 +32,29 @@ func TestRecoverState_ClusterPolicyRoundtrip(t *testing.T) { "the persisted policy must be recovered verbatim into FSMState") } +func TestRecoverStatePublishesRecoveredState(t *testing.T) { + t.Parallel() + + machine, store, _ := newTestMachineWithThreshold(t, 1000) + policy := &commonpb.ClusterPolicy{ + Revision: 4, + CapabilityActivations: []*commonpb.CapabilityActivation{{Id: 17, Active: true}}, + } + batch := store.OpenWriteSession() + require.NoError(t, SaveClusterPolicy(batch, policy)) + require.NoError(t, batch.Commit()) + + var recovered *FSMState + recovery := NewRecovery(machine, store, WithStateRecovered(func(state *FSMState) error { + recovered = state + + return nil + })) + require.NoError(t, recovery.RecoverState()) + require.NotNil(t, recovered) + require.True(t, policy.EqualVT(recovered.ClusterPolicy)) +} + // NewFSMState seeds a non-nil revision-0 default so the apply path and the // readiness gate can read the policy before any is committed. func TestNewFSMState_ClusterPolicyDefault(t *testing.T) { diff --git a/internal/infra/state/machine.go b/internal/infra/state/machine.go index cd647fcbc5..3357fc3390 100644 --- a/internal/infra/state/machine.go +++ b/internal/infra/state/machine.go @@ -159,9 +159,15 @@ type Machine struct { // cluster membership atomic with FSM state and makes the spool/WAL // replay path naturally idempotent — every replay of the entry // re-asserts the membership row. EN-1413. - confChangeHandler func(entry *raftpb.Entry, session *dal.WriteSession) error + confChangeHandler func(entry *raftpb.Entry, session *dal.WriteSession) error + capabilityPolicyCommitHandler func(*commonpb.ClusterPolicy) } +// NoopCapabilityPolicyCommitHandler is an explicit dependency for callers +// that do not maintain an in-memory membership gate (for example replay and +// isolated FSM tests). Production bootstrap injects the membership publisher. +func NoopCapabilityPolicyCommitHandler(*commonpb.ClusterPolicy) {} + // NewMachine constructs the hot-path FSM. It composes pre-built sub-objects // (Registry, CacheSnapshotter) so the constructor reads no Pebble and keeps // the surface focused on hot-path concerns. The bootstrap is responsible for @@ -172,7 +178,11 @@ type Machine struct { // confChangeHandler is invoked from PrepareEntries for every // EntryConfChange* in the in-flight batch (see the field comment on // Machine). Must be non-nil. -func NewMachine(logger logging.Logger, registry *StateRegistry, cacheSnapshotter *CacheSnapshotter, fileArtifacts dal.QueryCheckpoints, sentinel dal.SentinelFactory, meterProvider metric.MeterProvider, ks *keystore.KeyStore, sharedState *SharedState, notifier Notifier, bloomFilters *bloom.FilterSet, auditKey string, numscriptCacheSize int, confChangeHandler func(entry *raftpb.Entry, session *dal.WriteSession) error) (*Machine, error) { +func NewMachine(logger logging.Logger, registry *StateRegistry, cacheSnapshotter *CacheSnapshotter, fileArtifacts dal.QueryCheckpoints, sentinel dal.SentinelFactory, meterProvider metric.MeterProvider, ks *keystore.KeyStore, sharedState *SharedState, notifier Notifier, bloomFilters *bloom.FilterSet, auditKey string, numscriptCacheSize int, confChangeHandler func(entry *raftpb.Entry, session *dal.WriteSession) error, capabilityPolicyCommitHandler func(*commonpb.ClusterPolicy)) (*Machine, error) { + if capabilityPolicyCommitHandler == nil { + return nil, errors.New("capability policy commit handler is required") + } + sentinelMode := sentinel.IsEnabled() // raft.* metrics describe the consensus engine and follow the // upstream etcd-raft naming convention; numscript.* metrics are @@ -247,6 +257,7 @@ func NewMachine(logger logging.Logger, registry *StateRegistry, cacheSnapshotter bloomRebuildCh: make(chan string, 1), auditHashBuf: make([]byte, 0, 4096), confChangeHandler: confChangeHandler, + capabilityPolicyCommitHandler: capabilityPolicyCommitHandler, } fsm.appliedCond = sync.NewCond(&fsm.appliedMu) fsm.cacheSnapshotter = cacheSnapshotter @@ -315,6 +326,16 @@ func (fsm *Machine) LastAppliedIndex() uint64 { return fsm.State.LastAppliedIndex } +// ClusterPolicy returns the policy state at the current apply position. It is +// intended for callbacks invoked while PrepareEntries holds fsm.mu. +func (fsm *Machine) ClusterPolicy() *commonpb.ClusterPolicy { + if fsm.State.ClusterPolicy == nil { + return nil + } + + return fsm.State.ClusterPolicy.CloneVT() +} + // RestoreState atomically replaces the FSM-level state. The intended callers // are Recovery (at boot) and Synchronizer (after install-snapshot), which // build a fresh FSMState from Pebble via LoadFSMStateFromStore and swap it @@ -799,6 +820,9 @@ func (fsm *Machine) PrepareDecodedEntries(ctx context.Context, sessions dal.Writ bloomReplacementEpoch: bloomReplacementEpoch, entryCount: len(decoded), } + if fsm.State.ClusterPolicy != nil { + pb.capabilityPolicy = fsm.State.ClusterPolicy.CloneVT() + } // Capture sentinel data before releasing the lock. if fsm.sentinelMode { @@ -836,6 +860,9 @@ func (fsm *Machine) CommitPreparedBatch(ctx context.Context, pb *PreparedBatch) if pb.bloomReplacement { fsm.BloomFilters.ConfirmPersistedReplacement(pb.bloomReplacementEpoch) } + if fsm.capabilityPolicyCommitHandler != nil && pb.capabilityPolicy != nil { + fsm.capabilityPolicyCommitHandler(pb.capabilityPolicy) + } fsm.batchCommitHistogram.Record(ctx, time.Since(commitStart).Seconds()) @@ -2080,6 +2107,7 @@ type PreparedBatch struct { sentinelLedgerNames []string sentinelDeletedLedgerNames map[string]struct{} sentinelTracer *SentinelTracer + capabilityPolicy *commonpb.ClusterPolicy entryCount int } diff --git a/internal/infra/state/machine_test.go b/internal/infra/state/machine_test.go index 9e0f24cdf1..f14cdda368 100644 --- a/internal/infra/state/machine_test.go +++ b/internal/infra/state/machine_test.go @@ -49,7 +49,7 @@ func newTestMachineWithThreshold(t *testing.T, generationThreshold uint64) (*Mac registry := NewStateRegistry(c, attrs) snapshotter := NewCacheSnapshotter(logger, registry, nil) - machine, err := NewMachine(logger, registry, snapshotter, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler) + machine, err := NewMachine(logger, registry, snapshotter, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler, NoopCapabilityPolicyCommitHandler) require.NoError(t, err) // NewMachine no longer performs the initial recoverState — the caller is @@ -113,7 +113,7 @@ func recoverMachineOnStore(t *testing.T, dataStore *dal.Store) *Machine { registry := NewStateRegistry(c, attributes.New()) snapshotter := NewCacheSnapshotter(logger, registry, nil) - m, err := NewMachine(logger, registry, snapshotter, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler) + m, err := NewMachine(logger, registry, snapshotter, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler, NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, NewRecovery(m, dataStore).RecoverState()) @@ -922,7 +922,7 @@ func TestNextLedgerIDRecovery(t *testing.T) { // Create first machine and 3 ledgers. reg1 := NewStateRegistry(c, attrs) snap1 := NewCacheSnapshotter(logger, reg1, nil) - machine1, err := NewMachine(logger, reg1, snap1, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler) + machine1, err := NewMachine(logger, reg1, snap1, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler, NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, NewRecovery(machine1, dataStore).RecoverState()) @@ -958,7 +958,7 @@ func TestNextLedgerIDRecovery(t *testing.T) { reg2 := NewStateRegistry(c2, attrs2) snap2 := NewCacheSnapshotter(logger, reg2, nil) - machine2, err := NewMachine(logger, reg2, snap2, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler) + machine2, err := NewMachine(logger, reg2, snap2, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler, NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, NewRecovery(machine2, dataStore).RecoverState()) @@ -1018,7 +1018,7 @@ func TestPrepareEntriesTraceLogPipeliningLag(t *testing.T) { registry := NewStateRegistry(c, attributes.New()) snapshotter := NewCacheSnapshotter(logger, registry, nil) - machine, err := NewMachine(logger, registry, snapshotter, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler) + machine, err := NewMachine(logger, registry, snapshotter, dataStore, dal.NewSentinelFactory(dataStore, false), meterProvider, keystore.NewKeyStore(), NewSharedState(), newNoopNotifier(t), nil, "test-cluster", 0, noopConfChangeHandler, NoopCapabilityPolicyCommitHandler) require.NoError(t, err) require.NoError(t, NewRecovery(machine, dataStore).RecoverState()) diff --git a/internal/infra/state/recovery.go b/internal/infra/state/recovery.go index 6796098a9a..8a3d46e8a4 100644 --- a/internal/infra/state/recovery.go +++ b/internal/infra/state/recovery.go @@ -18,14 +18,34 @@ import ( // it uses, and a future contributor reading machine.go won't find any read // primitive to pull from. type Recovery struct { - apply *Machine - reader dal.RecoveryReader + apply *Machine + reader dal.RecoveryReader + onStateRecovered func(*FSMState) error +} + +// RecoveryOption customizes the post-recovery hooks without coupling the +// state package to any of its runtime consumers. +type RecoveryOption func(*Recovery) + +// WithStateRecovered registers a hook invoked after the FSM and its +// in-memory trackers have been rebuilt. Runtime consumers use this to +// republish derived state (for example, the membership capability gate) after +// boot or snapshot recovery. +func WithStateRecovered(hook func(*FSMState) error) RecoveryOption { + return func(recovery *Recovery) { + recovery.onStateRecovered = hook + } } // NewRecovery wires a Recovery onto an existing Machine. The reader is held // only on the Recovery, never on the Machine. -func NewRecovery(apply *Machine, reader dal.RecoveryReader) *Recovery { - return &Recovery{apply: apply, reader: reader} +func NewRecovery(apply *Machine, reader dal.RecoveryReader, opts ...RecoveryOption) *Recovery { + recovery := &Recovery{apply: apply, reader: reader} + for _, opt := range opts { + opt(recovery) + } + + return recovery } // RecoverState rebuilds the Machine from the Pebble data store, in three @@ -181,6 +201,12 @@ func (r *Recovery) RecoverState() error { "reversionLedgers": len(reversions), }).Infof("Recovered FSM state from store") + if r.onStateRecovered != nil { + if err := r.onStateRecovered(newState); err != nil { + return fmt.Errorf("publishing derived state after recovery: %w", err) + } + } + return nil } diff --git a/internal/infra/state/sentinel_tracer_isolation_test.go b/internal/infra/state/sentinel_tracer_isolation_test.go index 9f33f7b2df..fee8c57e15 100644 --- a/internal/infra/state/sentinel_tracer_isolation_test.go +++ b/internal/infra/state/sentinel_tracer_isolation_test.go @@ -49,6 +49,7 @@ func TestSentinelTracerIsolatedAcrossBatches(t *testing.T) { "test-cluster", 0, noopConfChangeHandler, + NoopCapabilityPolicyCommitHandler, ) require.NoError(t, err) require.NoError(t, NewRecovery(machine, dataStore).RecoverState()) diff --git a/internal/infra/state/signing_cascade_test.go b/internal/infra/state/signing_cascade_test.go index 2456e048a2..3ffe18ea92 100644 --- a/internal/infra/state/signing_cascade_test.go +++ b/internal/infra/state/signing_cascade_test.go @@ -257,6 +257,7 @@ func (r *signingBatchRunner) restart() *keystore.KeyStore { logger, registry, NewCacheSnapshotter(logger, registry, nil), r.store, dal.NewSentinelFactory(r.store, false), meterProvider, recovered, NewSharedState(), newNoopNotifier(r.t), nil, "test-cluster", 0, noopConfChangeHandler, + NoopCapabilityPolicyCommitHandler, ) require.NoError(r.t, err) require.NoError(r.t, NewRecovery(machine, r.store).RecoverState()) diff --git a/internal/pkg/reconciler/reconciler.go b/internal/pkg/reconciler/reconciler.go new file mode 100644 index 0000000000..d634947734 --- /dev/null +++ b/internal/pkg/reconciler/reconciler.go @@ -0,0 +1,68 @@ +// Package reconciler provides a small periodic reconciliation loop with an +// explicit trigger for events that should be handled before the next tick. +package reconciler + +import ( + "context" + "time" + + "github.com/formancehq/ledger/v3/internal/pkg/worker" +) + +// Reconciler invokes reconcile immediately when started, periodically at +// interval, and whenever Trigger is called. Trigger notifications are +// coalesced so bursts of events cannot grow an unbounded queue. +type Reconciler struct { + reconcile func(context.Context) + interval time.Duration + trigger chan struct{} + w worker.Worker +} + +// New creates a periodic reconciler. interval must be positive. +func New(reconcile func(context.Context), interval time.Duration) *Reconciler { + if interval <= 0 { + panic("reconciler interval must be positive") + } + + return &Reconciler{ + reconcile: reconcile, + interval: interval, + trigger: make(chan struct{}, 1), + w: worker.New(), + } +} + +// Trigger asks the running reconciler to converge immediately. The signal is +// coalesced because several events may be committed in one batch. +func (r *Reconciler) Trigger() { + select { + case r.trigger <- struct{}{}: + default: + } +} + +// Start runs an immediate reconciliation, then one per interval, until Stop. +func (r *Reconciler) Start() { + r.w.Run(func(stop <-chan struct{}) { + ctx := worker.ContextFromStop(stop) + r.reconcile(ctx) + ticker := time.NewTicker(r.interval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + r.reconcile(ctx) + case <-r.trigger: + r.reconcile(ctx) + } + } + }) +} + +// Stop signals the background goroutine to stop and waits for it to finish. +func (r *Reconciler) Stop() { + r.w.Stop() +} diff --git a/internal/pkg/reconciler/reconciler_test.go b/internal/pkg/reconciler/reconciler_test.go new file mode 100644 index 0000000000..8d61360c78 --- /dev/null +++ b/internal/pkg/reconciler/reconciler_test.go @@ -0,0 +1,55 @@ +package reconciler + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestReconcilerTicksUntilStopped(t *testing.T) { + t.Parallel() + + var count atomic.Int32 + r := New(func(context.Context) { count.Add(1) }, 5*time.Millisecond) + r.Start() + + require.Eventually(t, func() bool { return count.Load() >= 3 }, 2*time.Second, 5*time.Millisecond, + "reconcile must run repeatedly on the ticker") + + r.Stop() + settled := count.Load() + require.Never(t, func() bool { return count.Load() > settled }, 100*time.Millisecond, 10*time.Millisecond, + "no reconcile must run after Stop") +} + +func TestReconcilerTriggerIsCoalescedAndRaceSafe(t *testing.T) { + t.Parallel() + + var count atomic.Int32 + r := New(func(context.Context) { count.Add(1) }, time.Hour) + r.Start() + require.Eventually(t, func() bool { return count.Load() >= 1 }, time.Second, time.Millisecond) + + var wg sync.WaitGroup + for range 100 { + wg.Go(func() { + r.Trigger() + }) + } + wg.Wait() + require.Eventually(t, func() bool { return count.Load() >= 2 }, time.Second, time.Millisecond, + "an event must wake reconciliation without waiting for the ticker") + r.Stop() +} + +func TestNewRejectsNonPositiveInterval(t *testing.T) { + t.Parallel() + + require.Panics(t, func() { + New(func(context.Context) {}, 0) + }) +} diff --git a/internal/proto/clusterbootstrappb/cluster_bootstrap.pb.go b/internal/proto/clusterbootstrappb/cluster_bootstrap.pb.go index 9677fa02bb..2da477a78f 100644 --- a/internal/proto/clusterbootstrappb/cluster_bootstrap.pb.go +++ b/internal/proto/clusterbootstrappb/cluster_bootstrap.pb.go @@ -109,6 +109,7 @@ type PeerInfo struct { RaftAddress string `protobuf:"bytes,2,opt,name=raft_address,json=raftAddress,proto3" json:"raft_address,omitempty"` ServiceAddress string `protobuf:"bytes,3,opt,name=service_address,json=serviceAddress,proto3" json:"service_address,omitempty"` InstanceId []byte `protobuf:"bytes,4,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // 16-byte UUID identifying this peer's WAL/PVC incarnation; required + Capabilities []uint32 `protobuf:"fixed32,5,rep,packed,name=capabilities,proto3" json:"capabilities,omitempty"` // capabilities supported by this binary unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -171,12 +172,20 @@ func (x *PeerInfo) GetInstanceId() []byte { return nil } +func (x *PeerInfo) GetCapabilities() []uint32 { + if x != nil { + return x.Capabilities + } + return nil +} + type JoinAsLearnerRequest struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId uint64 `protobuf:"varint,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` // Numeric ID for the new learner node RaftAddress string `protobuf:"bytes,2,opt,name=raft_address,json=raftAddress,proto3" json:"raft_address,omitempty"` // Raft transport address of the learner ServiceAddress string `protobuf:"bytes,3,opt,name=service_address,json=serviceAddress,proto3" json:"service_address,omitempty"` // Service API address of the learner InstanceId []byte `protobuf:"bytes,4,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // 16-byte UUID identifying this peer's WAL/PVC incarnation (EN-1045); required + Capabilities []uint32 `protobuf:"fixed32,5,rep,packed,name=capabilities,proto3" json:"capabilities,omitempty"` // capabilities supported by this binary unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -239,6 +248,13 @@ func (x *JoinAsLearnerRequest) GetInstanceId() []byte { return nil } +func (x *JoinAsLearnerRequest) GetCapabilities() []uint32 { + if x != nil { + return x.Capabilities + } + return nil +} + type JoinAsLearnerResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -275,6 +291,122 @@ func (*JoinAsLearnerResponse) Descriptor() ([]byte, []int) { return file_cluster_bootstrap_proto_rawDescGZIP(), []int{4} } +// RefreshMembership carries the current identity, endpoints and capability +// evidence of a member that already exists in the committed Raft membership. +// It is used after a normal restart, where the node must not re-run the +// learner-join path. +type RefreshMembershipRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NodeId uint64 `protobuf:"varint,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + RaftAddress string `protobuf:"bytes,2,opt,name=raft_address,json=raftAddress,proto3" json:"raft_address,omitempty"` + ServiceAddress string `protobuf:"bytes,3,opt,name=service_address,json=serviceAddress,proto3" json:"service_address,omitempty"` + InstanceId []byte `protobuf:"bytes,4,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + Capabilities []uint32 `protobuf:"fixed32,5,rep,packed,name=capabilities,proto3" json:"capabilities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RefreshMembershipRequest) Reset() { + *x = RefreshMembershipRequest{} + mi := &file_cluster_bootstrap_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RefreshMembershipRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RefreshMembershipRequest) ProtoMessage() {} + +func (x *RefreshMembershipRequest) ProtoReflect() protoreflect.Message { + mi := &file_cluster_bootstrap_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RefreshMembershipRequest.ProtoReflect.Descriptor instead. +func (*RefreshMembershipRequest) Descriptor() ([]byte, []int) { + return file_cluster_bootstrap_proto_rawDescGZIP(), []int{5} +} + +func (x *RefreshMembershipRequest) GetNodeId() uint64 { + if x != nil { + return x.NodeId + } + return 0 +} + +func (x *RefreshMembershipRequest) GetRaftAddress() string { + if x != nil { + return x.RaftAddress + } + return "" +} + +func (x *RefreshMembershipRequest) GetServiceAddress() string { + if x != nil { + return x.ServiceAddress + } + return "" +} + +func (x *RefreshMembershipRequest) GetInstanceId() []byte { + if x != nil { + return x.InstanceId + } + return nil +} + +func (x *RefreshMembershipRequest) GetCapabilities() []uint32 { + if x != nil { + return x.Capabilities + } + return nil +} + +type RefreshMembershipResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RefreshMembershipResponse) Reset() { + *x = RefreshMembershipResponse{} + mi := &file_cluster_bootstrap_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RefreshMembershipResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RefreshMembershipResponse) ProtoMessage() {} + +func (x *RefreshMembershipResponse) ProtoReflect() protoreflect.Message { + mi := &file_cluster_bootstrap_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RefreshMembershipResponse.ProtoReflect.Descriptor instead. +func (*RefreshMembershipResponse) Descriptor() ([]byte, []int) { + return file_cluster_bootstrap_proto_rawDescGZIP(), []int{6} +} + var File_cluster_bootstrap_proto protoreflect.FileDescriptor const file_cluster_bootstrap_proto_rawDesc = "" + @@ -282,23 +414,34 @@ const file_cluster_bootstrap_proto_rawDesc = "" + "\x17cluster_bootstrap.proto\x12\x11cluster_bootstrap\"\x11\n" + "\x0fGetPeersRequest\"E\n" + "\x10GetPeersResponse\x121\n" + - "\x05peers\x18\x01 \x03(\v2\x1b.cluster_bootstrap.PeerInfoR\x05peers\"\x87\x01\n" + + "\x05peers\x18\x01 \x03(\v2\x1b.cluster_bootstrap.PeerInfoR\x05peers\"\xab\x01\n" + "\bPeerInfo\x12\x0e\n" + "\x02id\x18\x01 \x01(\x04R\x02id\x12!\n" + "\fraft_address\x18\x02 \x01(\tR\vraftAddress\x12'\n" + "\x0fservice_address\x18\x03 \x01(\tR\x0eserviceAddress\x12\x1f\n" + "\vinstance_id\x18\x04 \x01(\fR\n" + - "instanceId\"\x9c\x01\n" + + "instanceId\x12\"\n" + + "\fcapabilities\x18\x05 \x03(\aR\fcapabilities\"\xc0\x01\n" + "\x14JoinAsLearnerRequest\x12\x17\n" + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12!\n" + "\fraft_address\x18\x02 \x01(\tR\vraftAddress\x12'\n" + "\x0fservice_address\x18\x03 \x01(\tR\x0eserviceAddress\x12\x1f\n" + "\vinstance_id\x18\x04 \x01(\fR\n" + - "instanceId\"\x17\n" + - "\x15JoinAsLearnerResponse2\xd2\x01\n" + + "instanceId\x12\"\n" + + "\fcapabilities\x18\x05 \x03(\aR\fcapabilities\"\x17\n" + + "\x15JoinAsLearnerResponse\"\xc4\x01\n" + + "\x18RefreshMembershipRequest\x12\x17\n" + + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12!\n" + + "\fraft_address\x18\x02 \x01(\tR\vraftAddress\x12'\n" + + "\x0fservice_address\x18\x03 \x01(\tR\x0eserviceAddress\x12\x1f\n" + + "\vinstance_id\x18\x04 \x01(\fR\n" + + "instanceId\x12\"\n" + + "\fcapabilities\x18\x05 \x03(\aR\fcapabilities\"\x1b\n" + + "\x19RefreshMembershipResponse2\xc2\x02\n" + "\x17ClusterBootstrapService\x12S\n" + "\bGetPeers\x12\".cluster_bootstrap.GetPeersRequest\x1a#.cluster_bootstrap.GetPeersResponse\x12b\n" + - "\rJoinAsLearner\x12'.cluster_bootstrap.JoinAsLearnerRequest\x1a(.cluster_bootstrap.JoinAsLearnerResponseBCZAgithub.com/formancehq/ledger/v3/internal/proto/clusterbootstrappbb\x06proto3" + "\rJoinAsLearner\x12'.cluster_bootstrap.JoinAsLearnerRequest\x1a(.cluster_bootstrap.JoinAsLearnerResponse\x12n\n" + + "\x11RefreshMembership\x12+.cluster_bootstrap.RefreshMembershipRequest\x1a,.cluster_bootstrap.RefreshMembershipResponseBCZAgithub.com/formancehq/ledger/v3/internal/proto/clusterbootstrappbb\x06proto3" var ( file_cluster_bootstrap_proto_rawDescOnce sync.Once @@ -312,22 +455,26 @@ func file_cluster_bootstrap_proto_rawDescGZIP() []byte { return file_cluster_bootstrap_proto_rawDescData } -var file_cluster_bootstrap_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_cluster_bootstrap_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_cluster_bootstrap_proto_goTypes = []any{ - (*GetPeersRequest)(nil), // 0: cluster_bootstrap.GetPeersRequest - (*GetPeersResponse)(nil), // 1: cluster_bootstrap.GetPeersResponse - (*PeerInfo)(nil), // 2: cluster_bootstrap.PeerInfo - (*JoinAsLearnerRequest)(nil), // 3: cluster_bootstrap.JoinAsLearnerRequest - (*JoinAsLearnerResponse)(nil), // 4: cluster_bootstrap.JoinAsLearnerResponse + (*GetPeersRequest)(nil), // 0: cluster_bootstrap.GetPeersRequest + (*GetPeersResponse)(nil), // 1: cluster_bootstrap.GetPeersResponse + (*PeerInfo)(nil), // 2: cluster_bootstrap.PeerInfo + (*JoinAsLearnerRequest)(nil), // 3: cluster_bootstrap.JoinAsLearnerRequest + (*JoinAsLearnerResponse)(nil), // 4: cluster_bootstrap.JoinAsLearnerResponse + (*RefreshMembershipRequest)(nil), // 5: cluster_bootstrap.RefreshMembershipRequest + (*RefreshMembershipResponse)(nil), // 6: cluster_bootstrap.RefreshMembershipResponse } var file_cluster_bootstrap_proto_depIdxs = []int32{ 2, // 0: cluster_bootstrap.GetPeersResponse.peers:type_name -> cluster_bootstrap.PeerInfo 0, // 1: cluster_bootstrap.ClusterBootstrapService.GetPeers:input_type -> cluster_bootstrap.GetPeersRequest 3, // 2: cluster_bootstrap.ClusterBootstrapService.JoinAsLearner:input_type -> cluster_bootstrap.JoinAsLearnerRequest - 1, // 3: cluster_bootstrap.ClusterBootstrapService.GetPeers:output_type -> cluster_bootstrap.GetPeersResponse - 4, // 4: cluster_bootstrap.ClusterBootstrapService.JoinAsLearner:output_type -> cluster_bootstrap.JoinAsLearnerResponse - 3, // [3:5] is the sub-list for method output_type - 1, // [1:3] is the sub-list for method input_type + 5, // 3: cluster_bootstrap.ClusterBootstrapService.RefreshMembership:input_type -> cluster_bootstrap.RefreshMembershipRequest + 1, // 4: cluster_bootstrap.ClusterBootstrapService.GetPeers:output_type -> cluster_bootstrap.GetPeersResponse + 4, // 5: cluster_bootstrap.ClusterBootstrapService.JoinAsLearner:output_type -> cluster_bootstrap.JoinAsLearnerResponse + 6, // 6: cluster_bootstrap.ClusterBootstrapService.RefreshMembership:output_type -> cluster_bootstrap.RefreshMembershipResponse + 4, // [4:7] is the sub-list for method output_type + 1, // [1:4] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name 1, // [1:1] is the sub-list for extension extendee 0, // [0:1] is the sub-list for field type_name @@ -344,7 +491,7 @@ func file_cluster_bootstrap_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cluster_bootstrap_proto_rawDesc), len(file_cluster_bootstrap_proto_rawDesc)), NumEnums: 0, - NumMessages: 5, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/clusterbootstrappb/cluster_bootstrap_dethash.pb.go b/internal/proto/clusterbootstrappb/cluster_bootstrap_dethash.pb.go index 4df9fe2ba2..fa13058669 100644 --- a/internal/proto/clusterbootstrappb/cluster_bootstrap_dethash.pb.go +++ b/internal/proto/clusterbootstrappb/cluster_bootstrap_dethash.pb.go @@ -57,3 +57,25 @@ func (m *JoinAsLearnerResponse) MarshalDeterministicVT(dAtA []byte) []byte { } return append(dAtA, b...) } + +func (m *RefreshMembershipRequest) MarshalDeterministicVT(dAtA []byte) []byte { + if m == nil { + return dAtA + } + b, err := m.MarshalVT() + if err != nil { + panic("MarshalDeterministicVT: " + err.Error()) + } + return append(dAtA, b...) +} + +func (m *RefreshMembershipResponse) MarshalDeterministicVT(dAtA []byte) []byte { + if m == nil { + return dAtA + } + b, err := m.MarshalVT() + if err != nil { + panic("MarshalDeterministicVT: " + err.Error()) + } + return append(dAtA, b...) +} diff --git a/internal/proto/clusterbootstrappb/cluster_bootstrap_grpc.pb.go b/internal/proto/clusterbootstrappb/cluster_bootstrap_grpc.pb.go index e4fe414fd1..f626353548 100644 --- a/internal/proto/clusterbootstrappb/cluster_bootstrap_grpc.pb.go +++ b/internal/proto/clusterbootstrappb/cluster_bootstrap_grpc.pb.go @@ -19,8 +19,9 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - ClusterBootstrapService_GetPeers_FullMethodName = "/cluster_bootstrap.ClusterBootstrapService/GetPeers" - ClusterBootstrapService_JoinAsLearner_FullMethodName = "/cluster_bootstrap.ClusterBootstrapService/JoinAsLearner" + ClusterBootstrapService_GetPeers_FullMethodName = "/cluster_bootstrap.ClusterBootstrapService/GetPeers" + ClusterBootstrapService_JoinAsLearner_FullMethodName = "/cluster_bootstrap.ClusterBootstrapService/JoinAsLearner" + ClusterBootstrapService_RefreshMembership_FullMethodName = "/cluster_bootstrap.ClusterBootstrapService/RefreshMembership" ) // ClusterBootstrapServiceClient is the client API for ClusterBootstrapService service. @@ -31,7 +32,7 @@ const ( // // Unlike ClusterService (exposed on the external ServiceServer behind the // user JWT/scope pipeline), this service lives on the internal RaftServer -// alongside the Raft transport. It carries the two RPCs required by a node +// alongside the Raft transport. It carries the RPCs required by a node // that is joining an existing cluster: // // 1. GetPeers — discover the current set of voter nodes and their @@ -39,6 +40,8 @@ const ( // snapshot before starting Raft. // 2. JoinAsLearner — register with the leader as a new learner, which // proposes the corresponding ConfChange. +// 3. RefreshMembership — refresh the committed endpoints and capability +// evidence of an existing member after restart. // // Authentication is the RaftServer's own: cluster-id metadata (always) and // the cluster-secret bearer when configured. No user JWT or scope check @@ -46,6 +49,7 @@ const ( type ClusterBootstrapServiceClient interface { GetPeers(ctx context.Context, in *GetPeersRequest, opts ...grpc.CallOption) (*GetPeersResponse, error) JoinAsLearner(ctx context.Context, in *JoinAsLearnerRequest, opts ...grpc.CallOption) (*JoinAsLearnerResponse, error) + RefreshMembership(ctx context.Context, in *RefreshMembershipRequest, opts ...grpc.CallOption) (*RefreshMembershipResponse, error) } type clusterBootstrapServiceClient struct { @@ -76,6 +80,16 @@ func (c *clusterBootstrapServiceClient) JoinAsLearner(ctx context.Context, in *J return out, nil } +func (c *clusterBootstrapServiceClient) RefreshMembership(ctx context.Context, in *RefreshMembershipRequest, opts ...grpc.CallOption) (*RefreshMembershipResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RefreshMembershipResponse) + err := c.cc.Invoke(ctx, ClusterBootstrapService_RefreshMembership_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // ClusterBootstrapServiceServer is the server API for ClusterBootstrapService service. // All implementations must embed UnimplementedClusterBootstrapServiceServer // for forward compatibility. @@ -84,7 +98,7 @@ func (c *clusterBootstrapServiceClient) JoinAsLearner(ctx context.Context, in *J // // Unlike ClusterService (exposed on the external ServiceServer behind the // user JWT/scope pipeline), this service lives on the internal RaftServer -// alongside the Raft transport. It carries the two RPCs required by a node +// alongside the Raft transport. It carries the RPCs required by a node // that is joining an existing cluster: // // 1. GetPeers — discover the current set of voter nodes and their @@ -92,6 +106,8 @@ func (c *clusterBootstrapServiceClient) JoinAsLearner(ctx context.Context, in *J // snapshot before starting Raft. // 2. JoinAsLearner — register with the leader as a new learner, which // proposes the corresponding ConfChange. +// 3. RefreshMembership — refresh the committed endpoints and capability +// evidence of an existing member after restart. // // Authentication is the RaftServer's own: cluster-id metadata (always) and // the cluster-secret bearer when configured. No user JWT or scope check @@ -99,6 +115,7 @@ func (c *clusterBootstrapServiceClient) JoinAsLearner(ctx context.Context, in *J type ClusterBootstrapServiceServer interface { GetPeers(context.Context, *GetPeersRequest) (*GetPeersResponse, error) JoinAsLearner(context.Context, *JoinAsLearnerRequest) (*JoinAsLearnerResponse, error) + RefreshMembership(context.Context, *RefreshMembershipRequest) (*RefreshMembershipResponse, error) mustEmbedUnimplementedClusterBootstrapServiceServer() } @@ -115,6 +132,9 @@ func (UnimplementedClusterBootstrapServiceServer) GetPeers(context.Context, *Get func (UnimplementedClusterBootstrapServiceServer) JoinAsLearner(context.Context, *JoinAsLearnerRequest) (*JoinAsLearnerResponse, error) { return nil, status.Error(codes.Unimplemented, "method JoinAsLearner not implemented") } +func (UnimplementedClusterBootstrapServiceServer) RefreshMembership(context.Context, *RefreshMembershipRequest) (*RefreshMembershipResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RefreshMembership not implemented") +} func (UnimplementedClusterBootstrapServiceServer) mustEmbedUnimplementedClusterBootstrapServiceServer() { } func (UnimplementedClusterBootstrapServiceServer) testEmbeddedByValue() {} @@ -173,6 +193,24 @@ func _ClusterBootstrapService_JoinAsLearner_Handler(srv interface{}, ctx context return interceptor(ctx, in, info, handler) } +func _ClusterBootstrapService_RefreshMembership_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RefreshMembershipRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ClusterBootstrapServiceServer).RefreshMembership(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ClusterBootstrapService_RefreshMembership_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ClusterBootstrapServiceServer).RefreshMembership(ctx, req.(*RefreshMembershipRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ClusterBootstrapService_ServiceDesc is the grpc.ServiceDesc for ClusterBootstrapService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -188,6 +226,10 @@ var ClusterBootstrapService_ServiceDesc = grpc.ServiceDesc{ MethodName: "JoinAsLearner", Handler: _ClusterBootstrapService_JoinAsLearner_Handler, }, + { + MethodName: "RefreshMembership", + Handler: _ClusterBootstrapService_RefreshMembership_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "cluster_bootstrap.proto", diff --git a/internal/proto/clusterbootstrappb/cluster_bootstrap_reader.pb.go b/internal/proto/clusterbootstrappb/cluster_bootstrap_reader.pb.go index e59dbe251f..f52cbfadf3 100644 --- a/internal/proto/clusterbootstrappb/cluster_bootstrap_reader.pb.go +++ b/internal/proto/clusterbootstrappb/cluster_bootstrap_reader.pb.go @@ -5,6 +5,7 @@ package clusterbootstrappb import ( bytes "bytes" + slices "slices" ) // GetPeersRequestReader provides read-only access to GetPeersRequest. @@ -143,6 +144,7 @@ type PeerInfoReader interface { GetRaftAddress() string GetServiceAddress() string GetInstanceId() []byte + GetCapabilities() []uint32 Mutate() *PeerInfo } @@ -164,6 +166,10 @@ func (r *peerInfoReadonly) GetInstanceId() []byte { return bytes.Clone((*PeerInfo)(r).GetInstanceId()) } +func (r *peerInfoReadonly) GetCapabilities() []uint32 { + return slices.Clone((*PeerInfo)(r).GetCapabilities()) +} + func (r *peerInfoReadonly) Mutate() *PeerInfo { return (*PeerInfo)(r).CloneVT() } @@ -223,6 +229,7 @@ type JoinAsLearnerRequestReader interface { GetRaftAddress() string GetServiceAddress() string GetInstanceId() []byte + GetCapabilities() []uint32 Mutate() *JoinAsLearnerRequest } @@ -244,6 +251,10 @@ func (r *joinAsLearnerRequestReadonly) GetInstanceId() []byte { return bytes.Clone((*JoinAsLearnerRequest)(r).GetInstanceId()) } +func (r *joinAsLearnerRequestReadonly) GetCapabilities() []uint32 { + return slices.Clone((*JoinAsLearnerRequest)(r).GetCapabilities()) +} + func (r *joinAsLearnerRequestReadonly) Mutate() *JoinAsLearnerRequest { return (*JoinAsLearnerRequest)(r).CloneVT() } @@ -359,3 +370,152 @@ func (l joinAsLearnerResponseListReadonly) Range(yield func(int, JoinAsLearnerRe func NewJoinAsLearnerResponseListReader(s []*JoinAsLearnerResponse) JoinAsLearnerResponseListReader { return joinAsLearnerResponseListReadonly(s) } + +// RefreshMembershipRequestReader provides read-only access to RefreshMembershipRequest. +// Call Mutate() to obtain a mutable clone. +type RefreshMembershipRequestReader interface { + GetNodeId() uint64 + GetRaftAddress() string + GetServiceAddress() string + GetInstanceId() []byte + GetCapabilities() []uint32 + Mutate() *RefreshMembershipRequest +} + +type refreshMembershipRequestReadonly RefreshMembershipRequest + +func (r *refreshMembershipRequestReadonly) GetNodeId() uint64 { + return (*RefreshMembershipRequest)(r).GetNodeId() +} + +func (r *refreshMembershipRequestReadonly) GetRaftAddress() string { + return (*RefreshMembershipRequest)(r).GetRaftAddress() +} + +func (r *refreshMembershipRequestReadonly) GetServiceAddress() string { + return (*RefreshMembershipRequest)(r).GetServiceAddress() +} + +func (r *refreshMembershipRequestReadonly) GetInstanceId() []byte { + return bytes.Clone((*RefreshMembershipRequest)(r).GetInstanceId()) +} + +func (r *refreshMembershipRequestReadonly) GetCapabilities() []uint32 { + return slices.Clone((*RefreshMembershipRequest)(r).GetCapabilities()) +} + +func (r *refreshMembershipRequestReadonly) Mutate() *RefreshMembershipRequest { + return (*RefreshMembershipRequest)(r).CloneVT() +} + +// AsReader returns a read-only view of this RefreshMembershipRequest. +func (m *RefreshMembershipRequest) AsReader() RefreshMembershipRequestReader { + if m == nil { + return nil + } + return (*refreshMembershipRequestReadonly)(m) +} + +// Mutate returns a mutable deep clone of this RefreshMembershipRequest. +func (m *RefreshMembershipRequest) Mutate() *RefreshMembershipRequest { + return m.CloneVT() +} + +// RefreshMembershipRequestListReader provides read-only iteration over []*RefreshMembershipRequest. +type RefreshMembershipRequestListReader interface { + Len() int + Get(i int) RefreshMembershipRequestReader + Range(yield func(int, RefreshMembershipRequestReader) bool) +} + +type refreshMembershipRequestListReadonly []*RefreshMembershipRequest + +func (l refreshMembershipRequestListReadonly) Len() int { return len(l) } + +func (l refreshMembershipRequestListReadonly) Get(i int) RefreshMembershipRequestReader { + v := l[i] + if v == nil { + return nil + } + return v.AsReader() +} + +func (l refreshMembershipRequestListReadonly) Range(yield func(int, RefreshMembershipRequestReader) bool) { + for i, v := range l { + var r RefreshMembershipRequestReader + if v != nil { + r = v.AsReader() + } + if !yield(i, r) { + return + } + } +} + +// NewRefreshMembershipRequestListReader wraps s for read-only iteration. The returned +// view aliases the underlying slice; do not mutate s afterwards. +func NewRefreshMembershipRequestListReader(s []*RefreshMembershipRequest) RefreshMembershipRequestListReader { + return refreshMembershipRequestListReadonly(s) +} + +// RefreshMembershipResponseReader provides read-only access to RefreshMembershipResponse. +// Call Mutate() to obtain a mutable clone. +type RefreshMembershipResponseReader interface { + Mutate() *RefreshMembershipResponse +} + +type refreshMembershipResponseReadonly RefreshMembershipResponse + +func (r *refreshMembershipResponseReadonly) Mutate() *RefreshMembershipResponse { + return (*RefreshMembershipResponse)(r).CloneVT() +} + +// AsReader returns a read-only view of this RefreshMembershipResponse. +func (m *RefreshMembershipResponse) AsReader() RefreshMembershipResponseReader { + if m == nil { + return nil + } + return (*refreshMembershipResponseReadonly)(m) +} + +// Mutate returns a mutable deep clone of this RefreshMembershipResponse. +func (m *RefreshMembershipResponse) Mutate() *RefreshMembershipResponse { + return m.CloneVT() +} + +// RefreshMembershipResponseListReader provides read-only iteration over []*RefreshMembershipResponse. +type RefreshMembershipResponseListReader interface { + Len() int + Get(i int) RefreshMembershipResponseReader + Range(yield func(int, RefreshMembershipResponseReader) bool) +} + +type refreshMembershipResponseListReadonly []*RefreshMembershipResponse + +func (l refreshMembershipResponseListReadonly) Len() int { return len(l) } + +func (l refreshMembershipResponseListReadonly) Get(i int) RefreshMembershipResponseReader { + v := l[i] + if v == nil { + return nil + } + return v.AsReader() +} + +func (l refreshMembershipResponseListReadonly) Range(yield func(int, RefreshMembershipResponseReader) bool) { + for i, v := range l { + var r RefreshMembershipResponseReader + if v != nil { + r = v.AsReader() + } + if !yield(i, r) { + return + } + } +} + +// NewRefreshMembershipResponseListReader wraps s for read-only iteration. The returned +// view aliases the underlying slice; do not mutate s afterwards. +func NewRefreshMembershipResponseListReader(s []*RefreshMembershipResponse) RefreshMembershipResponseListReader { + return refreshMembershipResponseListReadonly(s) +} diff --git a/internal/proto/clusterbootstrappb/cluster_bootstrap_vtproto.pb.go b/internal/proto/clusterbootstrappb/cluster_bootstrap_vtproto.pb.go index 7487310391..8124847378 100644 --- a/internal/proto/clusterbootstrappb/cluster_bootstrap_vtproto.pb.go +++ b/internal/proto/clusterbootstrappb/cluster_bootstrap_vtproto.pb.go @@ -5,6 +5,7 @@ package clusterbootstrappb import ( + binary "encoding/binary" fmt "fmt" protohelpers "github.com/planetscale/vtprotobuf/protohelpers" proto "google.golang.org/protobuf/proto" @@ -71,6 +72,11 @@ func (m *PeerInfo) CloneVT() *PeerInfo { copy(tmpBytes, rhs) r.InstanceId = tmpBytes } + if rhs := m.Capabilities; rhs != nil { + tmpContainer := make([]uint32, len(rhs)) + copy(tmpContainer, rhs) + r.Capabilities = tmpContainer + } if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -95,6 +101,11 @@ func (m *JoinAsLearnerRequest) CloneVT() *JoinAsLearnerRequest { copy(tmpBytes, rhs) r.InstanceId = tmpBytes } + if rhs := m.Capabilities; rhs != nil { + tmpContainer := make([]uint32, len(rhs)) + copy(tmpContainer, rhs) + r.Capabilities = tmpContainer + } if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -122,6 +133,51 @@ func (m *JoinAsLearnerResponse) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *RefreshMembershipRequest) CloneVT() *RefreshMembershipRequest { + if m == nil { + return (*RefreshMembershipRequest)(nil) + } + r := new(RefreshMembershipRequest) + r.NodeId = m.NodeId + r.RaftAddress = m.RaftAddress + r.ServiceAddress = m.ServiceAddress + if rhs := m.InstanceId; rhs != nil { + tmpBytes := make([]byte, len(rhs)) + copy(tmpBytes, rhs) + r.InstanceId = tmpBytes + } + if rhs := m.Capabilities; rhs != nil { + tmpContainer := make([]uint32, len(rhs)) + copy(tmpContainer, rhs) + r.Capabilities = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *RefreshMembershipRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *RefreshMembershipResponse) CloneVT() *RefreshMembershipResponse { + if m == nil { + return (*RefreshMembershipResponse)(nil) + } + r := new(RefreshMembershipResponse) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *RefreshMembershipResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (this *GetPeersRequest) EqualVT(that *GetPeersRequest) bool { if this == that { return true @@ -189,6 +245,15 @@ func (this *PeerInfo) EqualVT(that *PeerInfo) bool { if string(this.InstanceId) != string(that.InstanceId) { return false } + if len(this.Capabilities) != len(that.Capabilities) { + return false + } + for i, vx := range this.Capabilities { + vy := that.Capabilities[i] + if vx != vy { + return false + } + } return string(this.unknownFields) == string(that.unknownFields) } @@ -217,6 +282,15 @@ func (this *JoinAsLearnerRequest) EqualVT(that *JoinAsLearnerRequest) bool { if string(this.InstanceId) != string(that.InstanceId) { return false } + if len(this.Capabilities) != len(that.Capabilities) { + return false + } + for i, vx := range this.Capabilities { + vy := that.Capabilities[i] + if vx != vy { + return false + } + } return string(this.unknownFields) == string(that.unknownFields) } @@ -243,6 +317,59 @@ func (this *JoinAsLearnerResponse) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *RefreshMembershipRequest) EqualVT(that *RefreshMembershipRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.NodeId != that.NodeId { + return false + } + if this.RaftAddress != that.RaftAddress { + return false + } + if this.ServiceAddress != that.ServiceAddress { + return false + } + if string(this.InstanceId) != string(that.InstanceId) { + return false + } + if len(this.Capabilities) != len(that.Capabilities) { + return false + } + for i, vx := range this.Capabilities { + vy := that.Capabilities[i] + if vx != vy { + return false + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *RefreshMembershipRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*RefreshMembershipRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *RefreshMembershipResponse) EqualVT(that *RefreshMembershipResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *RefreshMembershipResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*RefreshMembershipResponse) + if !ok { + return false + } + return this.EqualVT(that) +} func (m *GetPeersRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -351,6 +478,15 @@ func (m *PeerInfo) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Capabilities) > 0 { + for iNdEx := len(m.Capabilities) - 1; iNdEx >= 0; iNdEx-- { + i -= 4 + binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Capabilities[iNdEx])) + } + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Capabilities)*4)) + i-- + dAtA[i] = 0x2a + } if len(m.InstanceId) > 0 { i -= len(m.InstanceId) copy(dAtA[i:], m.InstanceId) @@ -410,6 +546,15 @@ func (m *JoinAsLearnerRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Capabilities) > 0 { + for iNdEx := len(m.Capabilities) - 1; iNdEx >= 0; iNdEx-- { + i -= 4 + binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Capabilities[iNdEx])) + } + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Capabilities)*4)) + i-- + dAtA[i] = 0x2a + } if len(m.InstanceId) > 0 { i -= len(m.InstanceId) copy(dAtA[i:], m.InstanceId) @@ -472,6 +617,107 @@ func (m *JoinAsLearnerResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *RefreshMembershipRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RefreshMembershipRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RefreshMembershipRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Capabilities) > 0 { + for iNdEx := len(m.Capabilities) - 1; iNdEx >= 0; iNdEx-- { + i -= 4 + binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Capabilities[iNdEx])) + } + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Capabilities)*4)) + i-- + dAtA[i] = 0x2a + } + if len(m.InstanceId) > 0 { + i -= len(m.InstanceId) + copy(dAtA[i:], m.InstanceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.InstanceId))) + i-- + dAtA[i] = 0x22 + } + if len(m.ServiceAddress) > 0 { + i -= len(m.ServiceAddress) + copy(dAtA[i:], m.ServiceAddress) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ServiceAddress))) + i-- + dAtA[i] = 0x1a + } + if len(m.RaftAddress) > 0 { + i -= len(m.RaftAddress) + copy(dAtA[i:], m.RaftAddress) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RaftAddress))) + i-- + dAtA[i] = 0x12 + } + if m.NodeId != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.NodeId)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *RefreshMembershipResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RefreshMembershipResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RefreshMembershipResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + func (m *GetPeersRequest) SizeVT() (n int) { if m == nil { return 0 @@ -519,6 +765,9 @@ func (m *PeerInfo) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if len(m.Capabilities) > 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(len(m.Capabilities)*4)) + len(m.Capabilities)*4 + } n += len(m.unknownFields) return n } @@ -544,6 +793,9 @@ func (m *JoinAsLearnerRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if len(m.Capabilities) > 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(len(m.Capabilities)*4)) + len(m.Capabilities)*4 + } n += len(m.unknownFields) return n } @@ -558,6 +810,44 @@ func (m *JoinAsLearnerResponse) SizeVT() (n int) { return n } +func (m *RefreshMembershipRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.NodeId != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.NodeId)) + } + l = len(m.RaftAddress) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.ServiceAddress) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.InstanceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Capabilities) > 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(len(m.Capabilities)*4)) + len(m.Capabilities)*4 + } + n += len(m.unknownFields) + return n +} + +func (m *RefreshMembershipResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + func (m *GetPeersRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -840,6 +1130,58 @@ func (m *PeerInfo) UnmarshalVT(dAtA []byte) error { m.InstanceId = []byte{} } iNdEx = postIndex + case 5: + if wireType == 5 { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + elementCount = packedLen / 4 + if elementCount != 0 && len(m.Capabilities) == 0 { + m.Capabilities = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Capabilities", wireType) + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -1008,6 +1350,58 @@ func (m *JoinAsLearnerRequest) UnmarshalVT(dAtA []byte) error { m.InstanceId = []byte{} } iNdEx = postIndex + case 5: + if wireType == 5 { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + elementCount = packedLen / 4 + if elementCount != 0 && len(m.Capabilities) == 0 { + m.Capabilities = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Capabilities", wireType) + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -1081,3 +1475,274 @@ func (m *JoinAsLearnerResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *RefreshMembershipRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RefreshMembershipRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RefreshMembershipRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeId", wireType) + } + m.NodeId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeId |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RaftAddress", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RaftAddress = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ServiceAddress", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ServiceAddress = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InstanceId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InstanceId = append(m.InstanceId[:0], dAtA[iNdEx:postIndex]...) + if m.InstanceId == nil { + m.InstanceId = []byte{} + } + iNdEx = postIndex + case 5: + if wireType == 5 { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + elementCount = packedLen / 4 + if elementCount != 0 && len(m.Capabilities) == 0 { + m.Capabilities = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Capabilities", wireType) + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RefreshMembershipResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RefreshMembershipResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RefreshMembershipResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/internal/proto/clusterpb/cluster.pb.go b/internal/proto/clusterpb/cluster.pb.go index 1de4622a87..f090087bc5 100644 --- a/internal/proto/clusterpb/cluster.pb.go +++ b/internal/proto/clusterpb/cluster.pb.go @@ -1310,6 +1310,7 @@ type AddLearnerRequest struct { RaftAddress string `protobuf:"bytes,2,opt,name=raft_address,json=raftAddress,proto3" json:"raft_address,omitempty"` // Raft transport address of the learner ServiceAddress string `protobuf:"bytes,3,opt,name=service_address,json=serviceAddress,proto3" json:"service_address,omitempty"` // Service API address of the learner InstanceId []byte `protobuf:"bytes,4,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // 16-byte UUID from the learner's persisted INSTANCE_ID marker; required + Capabilities []uint32 `protobuf:"fixed32,5,rep,packed,name=capabilities,proto3" json:"capabilities,omitempty"` // Capabilities supported by the learner binary unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1372,6 +1373,13 @@ func (x *AddLearnerRequest) GetInstanceId() []byte { return nil } +func (x *AddLearnerRequest) GetCapabilities() []uint32 { + if x != nil { + return x.Capabilities + } + return nil +} + type AddLearnerResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -2524,13 +2532,14 @@ const file_cluster_proto_rawDesc = "" + "\n" + "wal_volume\x18\x01 \x01(\v2\x14.cluster.VolumeUsageR\twalVolume\x125\n" + "\vdata_volume\x18\x02 \x01(\v2\x14.cluster.VolumeUsageR\n" + - "dataVolume:\x04\xd8\xd5\"\x01\"\x99\x01\n" + + "dataVolume:\x04\xd8\xd5\"\x01\"\xbd\x01\n" + "\x11AddLearnerRequest\x12\x17\n" + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12!\n" + "\fraft_address\x18\x02 \x01(\tR\vraftAddress\x12'\n" + "\x0fservice_address\x18\x03 \x01(\tR\x0eserviceAddress\x12\x1f\n" + "\vinstance_id\x18\x04 \x01(\fR\n" + - "instanceId\"\x14\n" + + "instanceId\x12\"\n" + + "\fcapabilities\x18\x05 \x03(\aR\fcapabilities\"\x14\n" + "\x12AddLearnerResponse\"0\n" + "\x15PromoteLearnerRequest\x12\x17\n" + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\"\x18\n" + diff --git a/internal/proto/clusterpb/cluster_ledger_json.pb.go b/internal/proto/clusterpb/cluster_ledger_json.pb.go index dec8a02e65..523280bcd0 100644 --- a/internal/proto/clusterpb/cluster_ledger_json.pb.go +++ b/internal/proto/clusterpb/cluster_ledger_json.pb.go @@ -286,11 +286,18 @@ func (x *AddLearnerRequest) MarshalJSON() ([]byte, error) { RaftAddress string `json:"raftAddress"` ServiceAddress string `json:"serviceAddress"` InstanceId []byte `json:"instanceId"` + Capabilities any `json:"capabilities"` }{ NodeId: x.NodeId, RaftAddress: x.RaftAddress, ServiceAddress: x.ServiceAddress, InstanceId: x.InstanceId, + Capabilities: func() any { + if x.Capabilities == nil { + return []any{} + } + return x.Capabilities + }(), }) } diff --git a/internal/proto/clusterpb/cluster_reader.pb.go b/internal/proto/clusterpb/cluster_reader.pb.go index 4b488c478b..1754b53887 100644 --- a/internal/proto/clusterpb/cluster_reader.pb.go +++ b/internal/proto/clusterpb/cluster_reader.pb.go @@ -1660,6 +1660,7 @@ type AddLearnerRequestReader interface { GetRaftAddress() string GetServiceAddress() string GetInstanceId() []byte + GetCapabilities() []uint32 Mutate() *AddLearnerRequest } @@ -1681,6 +1682,10 @@ func (r *addLearnerRequestReadonly) GetInstanceId() []byte { return bytes.Clone((*AddLearnerRequest)(r).GetInstanceId()) } +func (r *addLearnerRequestReadonly) GetCapabilities() []uint32 { + return slices.Clone((*AddLearnerRequest)(r).GetCapabilities()) +} + func (r *addLearnerRequestReadonly) Mutate() *AddLearnerRequest { return (*AddLearnerRequest)(r).CloneVT() } diff --git a/internal/proto/clusterpb/cluster_vtproto.pb.go b/internal/proto/clusterpb/cluster_vtproto.pb.go index 08ec947843..36a91e11e2 100644 --- a/internal/proto/clusterpb/cluster_vtproto.pb.go +++ b/internal/proto/clusterpb/cluster_vtproto.pb.go @@ -443,6 +443,11 @@ func (m *AddLearnerRequest) CloneVT() *AddLearnerRequest { copy(tmpBytes, rhs) r.InstanceId = tmpBytes } + if rhs := m.Capabilities; rhs != nil { + tmpContainer := make([]uint32, len(rhs)) + copy(tmpContainer, rhs) + r.Capabilities = tmpContainer + } if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -1419,6 +1424,15 @@ func (this *AddLearnerRequest) EqualVT(that *AddLearnerRequest) bool { if string(this.InstanceId) != string(that.InstanceId) { return false } + if len(this.Capabilities) != len(that.Capabilities) { + return false + } + for i, vx := range this.Capabilities { + vy := that.Capabilities[i] + if vx != vy { + return false + } + } return string(this.unknownFields) == string(that.unknownFields) } @@ -3114,6 +3128,15 @@ func (m *AddLearnerRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Capabilities) > 0 { + for iNdEx := len(m.Capabilities) - 1; iNdEx >= 0; iNdEx-- { + i -= 4 + binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Capabilities[iNdEx])) + } + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Capabilities)*4)) + i-- + dAtA[i] = 0x2a + } if len(m.InstanceId) > 0 { i -= len(m.InstanceId) copy(dAtA[i:], m.InstanceId) @@ -4534,6 +4557,9 @@ func (m *AddLearnerRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if len(m.Capabilities) > 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(len(m.Capabilities)*4)) + len(m.Capabilities)*4 + } n += len(m.unknownFields) return n } @@ -7716,6 +7742,58 @@ func (m *AddLearnerRequest) UnmarshalVT(dAtA []byte) error { m.InstanceId = []byte{} } iNdEx = postIndex + case 5: + if wireType == 5 { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + elementCount = packedLen / 4 + if elementCount != 0 && len(m.Capabilities) == 0 { + m.Capabilities = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Capabilities", wireType) + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/internal/proto/commonpb/common.pb.go b/internal/proto/commonpb/common.pb.go index 15c3fc1fe5..d18201be0a 100644 --- a/internal/proto/commonpb/common.pb.go +++ b/internal/proto/commonpb/common.pb.go @@ -4461,8 +4461,10 @@ type ClusterPolicy struct { MetadataMaxValueBytes uint64 `protobuf:"fixed64,6,opt,name=metadata_max_value_bytes,json=metadataMaxValueBytes,proto3" json:"metadata_max_value_bytes,omitempty"` MetadataMaxEntityBytes uint64 `protobuf:"fixed64,7,opt,name=metadata_max_entity_bytes,json=metadataMaxEntityBytes,proto3" json:"metadata_max_entity_bytes,omitempty"` MetadataMaxCommandBytes uint64 `protobuf:"fixed64,8,opt,name=metadata_max_command_bytes,json=metadataMaxCommandBytes,proto3" json:"metadata_max_command_bytes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Capability activation is replicated policy, never a node-local flag. + CapabilityActivations []*CapabilityActivation `protobuf:"bytes,13,rep,name=capability_activations,json=capabilityActivations,proto3" json:"capability_activations,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ClusterPolicy) Reset() { @@ -4551,6 +4553,13 @@ func (x *ClusterPolicy) GetMetadataMaxCommandBytes() uint64 { return 0 } +func (x *ClusterPolicy) GetCapabilityActivations() []*CapabilityActivation { + if x != nil { + return x.CapabilityActivations + } + return nil +} + // CreatedPreparedQueryLog records the creation of a prepared query. type CreatedPreparedQueryLog struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -22169,6 +22178,60 @@ func (x *ListOptions) GetFilter() *QueryFilter { return nil } +// CapabilityActivation is intentionally small and wire-stable. IDs are fixed +// integers owned by the producer, not hashes of implementation names. +type CapabilityActivation struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id uint32 `protobuf:"fixed32,1,opt,name=id,proto3" json:"id,omitempty"` + Active bool `protobuf:"varint,2,opt,name=active,proto3" json:"active,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CapabilityActivation) Reset() { + *x = CapabilityActivation{} + mi := &file_common_proto_msgTypes[312] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CapabilityActivation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CapabilityActivation) ProtoMessage() {} + +func (x *CapabilityActivation) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[312] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CapabilityActivation.ProtoReflect.Descriptor instead. +func (*CapabilityActivation) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{312} +} + +func (x *CapabilityActivation) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *CapabilityActivation) GetActive() bool { + if x != nil { + return x.Active + } + return false +} + var file_common_proto_extTypes = []protoimpl.ExtensionInfo{ { ExtendedType: (*descriptorpb.MethodOptions)(nil), @@ -22544,7 +22607,7 @@ const file_common_proto_rawDesc = "" + "\x15PersistedClusterState\x12-\n" + "\x06config\x18\x01 \x01(\v2\x15.common.ClusterConfigR\x06config\x12\x1f\n" + "\vcache_epoch\x18\x02 \x01(\x06R\n" + - "cacheEpoch\"\xc9\x03\n" + + "cacheEpoch\"\x9e\x04\n" + "\rClusterPolicy\x12\x1a\n" + "\brevision\x18\x01 \x01(\x06R\brevision\x124\n" + "\x16idempotency_ttl_micros\x18\x02 \x01(\x06R\x14idempotencyTtlMicros\x124\n" + @@ -22553,7 +22616,8 @@ const file_common_proto_rawDesc = "" + "\x16metadata_max_key_bytes\x18\x05 \x01(\x06R\x13metadataMaxKeyBytes\x127\n" + "\x18metadata_max_value_bytes\x18\x06 \x01(\x06R\x15metadataMaxValueBytes\x129\n" + "\x19metadata_max_entity_bytes\x18\a \x01(\x06R\x16metadataMaxEntityBytes\x12;\n" + - "\x1ametadata_max_command_bytes\x18\b \x01(\x06R\x17metadataMaxCommandBytes:\x04\xd8\xd5\"\x01\"d\n" + + "\x1ametadata_max_command_bytes\x18\b \x01(\x06R\x17metadataMaxCommandBytes\x12S\n" + + "\x16capability_activations\x18\r \x03(\v2\x1c.common.CapabilityActivationR\x15capabilityActivations:\x04\xd8\xd5\"\x01\"d\n" + "\x17CreatedPreparedQueryLog\x12\x16\n" + "\x06ledger\x18\x01 \x01(\tR\x06ledger\x12+\n" + "\x05query\x18\x02 \x01(\v2\x15.common.PreparedQueryR\x05query:\x04\xd8\xd5\"\x01\"\xdc\x02\n" + @@ -23798,7 +23862,10 @@ const file_common_proto_rawDesc = "" + "\tpage_size\x18\x02 \x01(\rR\bpageSize\x12\x16\n" + "\x06cursor\x18\x03 \x01(\tR\x06cursor\x12\x18\n" + "\areverse\x18\x04 \x01(\bR\areverse\x12+\n" + - "\x06filter\x18\x05 \x01(\v2\x13.common.QueryFilterR\x06filter*\xbb\x03\n" + + "\x06filter\x18\x05 \x01(\v2\x13.common.QueryFilterR\x06filter\">\n" + + "\x14CapabilityActivation\x12\x0e\n" + + "\x02id\x18\x01 \x01(\aR\x02id\x12\x16\n" + + "\x06active\x18\x02 \x01(\bR\x06active*\xbb\x03\n" + "\tAuthScope\x12\x1a\n" + "\x16AUTH_SCOPE_UNSPECIFIED\x10\x00\x12\x1a\n" + "\x16AUTH_SCOPE_LEDGER_READ\x10\x01\x12\x1b\n" + @@ -24007,7 +24074,7 @@ func file_common_proto_rawDescGZIP() []byte { } var file_common_proto_enumTypes = make([]protoimpl.EnumInfo, 18) -var file_common_proto_msgTypes = make([]protoimpl.MessageInfo, 330) +var file_common_proto_msgTypes = make([]protoimpl.MessageInfo, 331) var file_common_proto_goTypes = []any{ (AuthScope)(0), // 0: common.AuthScope (DynamicAuthResolver)(0), // 1: common.DynamicAuthResolver @@ -24339,44 +24406,45 @@ var file_common_proto_goTypes = []any{ (*BackupStorage)(nil), // 327: common.BackupStorage (*ReadOptions)(nil), // 328: common.ReadOptions (*ListOptions)(nil), // 329: common.ListOptions - nil, // 330: common.MetadataMap.ValuesEntry - nil, // 331: common.Transaction.MetadataEntry - nil, // 332: common.Script.VarsEntry - nil, // 333: common.PostCommitVolumes.VolumesByAccountEntry - nil, // 334: common.Account.MetadataEntry - nil, // 335: common.MetadataSchema.AccountFieldsEntry - nil, // 336: common.MetadataSchema.TransactionFieldsEntry - nil, // 337: common.MetadataSchema.LedgerFieldsEntry - nil, // 338: common.SavedLedgerMetadataLog.MetadataEntry - nil, // 339: common.CreatedLedgerLog.AccountTypesEntry - nil, // 340: common.CreatedLedgerLog.MetadataEntry - nil, // 341: common.CreatedTransaction.AccountMetadataEntry - nil, // 342: common.SavedMetadata.MetadataEntry - nil, // 343: common.LedgerInfo.AccountTypesEntry - nil, // 344: common.LedgerInfo.MetadataEntry - nil, // 345: common.SaveMetadataCommand.MetadataEntry - nil, // 346: common.TransactionState.MetadataEntry - nil, // 347: common.AccountType.SegmentTypesEntry - (*signaturepb.SignedLog)(nil), // 348: signature.SignedLog - (*descriptorpb.MethodOptions)(nil), // 349: google.protobuf.MethodOptions - (*descriptorpb.MessageOptions)(nil), // 350: google.protobuf.MessageOptions - (*descriptorpb.FileOptions)(nil), // 351: google.protobuf.FileOptions - (*descriptorpb.FieldOptions)(nil), // 352: google.protobuf.FieldOptions + (*CapabilityActivation)(nil), // 330: common.CapabilityActivation + nil, // 331: common.MetadataMap.ValuesEntry + nil, // 332: common.Transaction.MetadataEntry + nil, // 333: common.Script.VarsEntry + nil, // 334: common.PostCommitVolumes.VolumesByAccountEntry + nil, // 335: common.Account.MetadataEntry + nil, // 336: common.MetadataSchema.AccountFieldsEntry + nil, // 337: common.MetadataSchema.TransactionFieldsEntry + nil, // 338: common.MetadataSchema.LedgerFieldsEntry + nil, // 339: common.SavedLedgerMetadataLog.MetadataEntry + nil, // 340: common.CreatedLedgerLog.AccountTypesEntry + nil, // 341: common.CreatedLedgerLog.MetadataEntry + nil, // 342: common.CreatedTransaction.AccountMetadataEntry + nil, // 343: common.SavedMetadata.MetadataEntry + nil, // 344: common.LedgerInfo.AccountTypesEntry + nil, // 345: common.LedgerInfo.MetadataEntry + nil, // 346: common.SaveMetadataCommand.MetadataEntry + nil, // 347: common.TransactionState.MetadataEntry + nil, // 348: common.AccountType.SegmentTypesEntry + (*signaturepb.SignedLog)(nil), // 349: signature.SignedLog + (*descriptorpb.MethodOptions)(nil), // 350: google.protobuf.MethodOptions + (*descriptorpb.MessageOptions)(nil), // 351: google.protobuf.MessageOptions + (*descriptorpb.FileOptions)(nil), // 352: google.protobuf.FileOptions + (*descriptorpb.FieldOptions)(nil), // 353: google.protobuf.FieldOptions } var file_common_proto_depIdxs = []int32{ 0, // 0: common.MethodAuthPolicy.fixed_scope:type_name -> common.AuthScope 1, // 1: common.MethodAuthPolicy.dynamic_resolver:type_name -> common.DynamicAuthResolver 20, // 2: common.MetadataValue.null_value:type_name -> common.NullValue - 330, // 3: common.MetadataMap.values:type_name -> common.MetadataMap.ValuesEntry + 331, // 3: common.MetadataMap.values:type_name -> common.MetadataMap.ValuesEntry 24, // 4: common.Posting.amount:type_name -> common.Uint256 25, // 5: common.Transaction.postings:type_name -> common.Posting - 331, // 6: common.Transaction.metadata:type_name -> common.Transaction.MetadataEntry + 332, // 6: common.Transaction.metadata:type_name -> common.Transaction.MetadataEntry 19, // 7: common.Transaction.timestamp:type_name -> common.Timestamp 19, // 8: common.Transaction.inserted_at:type_name -> common.Timestamp 19, // 9: common.Transaction.updated_at:type_name -> common.Timestamp 19, // 10: common.Transaction.reverted_at:type_name -> common.Timestamp 34, // 11: common.Transaction.post_commit_volumes:type_name -> common.PostCommitVolumes - 332, // 12: common.Script.vars:type_name -> common.Script.VarsEntry + 333, // 12: common.Script.vars:type_name -> common.Script.VarsEntry 28, // 13: common.SignedBigInt.magnitude:type_name -> common.BigUint 28, // 14: common.Volumes.input:type_name -> common.BigUint 28, // 15: common.Volumes.output:type_name -> common.BigUint @@ -24385,18 +24453,18 @@ var file_common_proto_depIdxs = []int32{ 29, // 18: common.VolumesWithBalance.balance:type_name -> common.SignedBigInt 33, // 19: common.VolumesByAssets.volumes:type_name -> common.VolumeEntry 30, // 20: common.VolumeEntry.volumes:type_name -> common.Volumes - 333, // 21: common.PostCommitVolumes.volumes_by_account:type_name -> common.PostCommitVolumes.VolumesByAccountEntry + 334, // 21: common.PostCommitVolumes.volumes_by_account:type_name -> common.PostCommitVolumes.VolumesByAccountEntry 31, // 22: common.AccountVolume.volumes:type_name -> common.VolumesWithBalance - 334, // 23: common.Account.metadata:type_name -> common.Account.MetadataEntry + 335, // 23: common.Account.metadata:type_name -> common.Account.MetadataEntry 19, // 24: common.Account.first_usage:type_name -> common.Timestamp 19, // 25: common.Account.insertion_date:type_name -> common.Timestamp 19, // 26: common.Account.updated_at:type_name -> common.Timestamp 35, // 27: common.Account.volumes:type_name -> common.AccountVolume 37, // 28: common.Target.account:type_name -> common.TargetAccount 3, // 29: common.MetadataFieldSchema.type:type_name -> common.MetadataType - 335, // 30: common.MetadataSchema.account_fields:type_name -> common.MetadataSchema.AccountFieldsEntry - 336, // 31: common.MetadataSchema.transaction_fields:type_name -> common.MetadataSchema.TransactionFieldsEntry - 337, // 32: common.MetadataSchema.ledger_fields:type_name -> common.MetadataSchema.LedgerFieldsEntry + 336, // 30: common.MetadataSchema.account_fields:type_name -> common.MetadataSchema.AccountFieldsEntry + 337, // 31: common.MetadataSchema.transaction_fields:type_name -> common.MetadataSchema.TransactionFieldsEntry + 338, // 32: common.MetadataSchema.ledger_fields:type_name -> common.MetadataSchema.LedgerFieldsEntry 2, // 33: common.SetMetadataFieldTypeCommand.target_type:type_name -> common.TargetType 3, // 34: common.SetMetadataFieldTypeCommand.type:type_name -> common.MetadataType 2, // 35: common.MetadataIndexID.target:type_name -> common.TargetType @@ -24407,7 +24475,7 @@ var file_common_proto_depIdxs = []int32{ 43, // 40: common.Index.id:type_name -> common.IndexID 19, // 41: common.Index.created_at:type_name -> common.Timestamp 47, // 42: common.Log.payload:type_name -> common.LogPayload - 348, // 43: common.Log.response_signature:type_name -> signature.SignedLog + 349, // 43: common.Log.response_signature:type_name -> signature.SignedLog 83, // 44: common.LogPayload.create_ledger:type_name -> common.CreatedLedgerLog 84, // 45: common.LogPayload.delete_ledger:type_name -> common.DeletedLedgerLog 85, // 46: common.LogPayload.apply:type_name -> common.ApplyLedgerLog @@ -24445,367 +24513,368 @@ var file_common_proto_depIdxs = []int32{ 57, // 78: common.ClusterConfig.bloom_prepared_queries:type_name -> common.BloomTypeConfig 57, // 79: common.ClusterConfig.bloom_indexes:type_name -> common.BloomTypeConfig 58, // 80: common.PersistedClusterState.config:type_name -> common.ClusterConfig - 312, // 81: common.CreatedPreparedQueryLog.query:type_name -> common.PreparedQuery - 292, // 82: common.UpdatedPreparedQueryLog.previous_filter:type_name -> common.QueryFilter - 292, // 83: common.UpdatedPreparedQueryLog.new_filter:type_name -> common.QueryFilter - 16, // 84: common.UpdatedPreparedQueryLog.previous_target:type_name -> common.QueryTarget - 16, // 85: common.UpdatedPreparedQueryLog.new_target:type_name -> common.QueryTarget - 338, // 86: common.SavedLedgerMetadataLog.metadata:type_name -> common.SavedLedgerMetadataLog.MetadataEntry - 19, // 87: common.NumscriptInfo.created_at:type_name -> common.Timestamp - 66, // 88: common.SavedNumscriptLog.info:type_name -> common.NumscriptInfo - 19, // 89: common.NumscriptVersionEntry.created_at:type_name -> common.Timestamp - 19, // 90: common.TemplateUsage.last_used:type_name -> common.Timestamp - 19, // 91: common.CreatedQueryCheckpointLog.created_at:type_name -> common.Timestamp - 77, // 92: common.SinkConfig.nats:type_name -> common.NatsSinkConfig - 78, // 93: common.SinkConfig.clickhouse:type_name -> common.ClickHouseSinkConfig - 79, // 94: common.SinkConfig.kafka:type_name -> common.KafkaSinkConfig - 80, // 95: common.SinkConfig.http:type_name -> common.HttpSinkConfig - 81, // 96: common.SinkConfig.databricks:type_name -> common.DatabricksSinkConfig - 8, // 97: common.SinkConfig.event_types:type_name -> common.EventType - 76, // 98: common.SinkStatus.error:type_name -> common.SinkError - 19, // 99: common.SinkError.occurred_at:type_name -> common.Timestamp - 82, // 100: common.DatabricksSinkConfig.oauth_m2m:type_name -> common.DatabricksOAuthM2M - 19, // 101: common.CreatedLedgerLog.created_at:type_name -> common.Timestamp - 40, // 102: common.CreatedLedgerLog.metadata_schema:type_name -> common.MetadataSchema - 9, // 103: common.CreatedLedgerLog.mode:type_name -> common.LedgerMode - 249, // 104: common.CreatedLedgerLog.mirror_source:type_name -> common.MirrorSourceConfig - 339, // 105: common.CreatedLedgerLog.account_types:type_name -> common.CreatedLedgerLog.AccountTypesEntry - 12, // 106: common.CreatedLedgerLog.default_enforcement_mode:type_name -> common.ChartEnforcementMode - 340, // 107: common.CreatedLedgerLog.metadata:type_name -> common.CreatedLedgerLog.MetadataEntry - 19, // 108: common.DeletedLedgerLog.deleted_at:type_name -> common.Timestamp - 86, // 109: common.ApplyLedgerLog.log:type_name -> common.LedgerLog - 88, // 110: common.LedgerLog.data:type_name -> common.LedgerLogPayload - 19, // 111: common.LedgerLog.date:type_name -> common.Timestamp - 87, // 112: common.LedgerLog.purged_volumes:type_name -> common.TouchedVolume - 87, // 113: common.LedgerLog.new_kept_volumes:type_name -> common.TouchedVolume - 87, // 114: common.LedgerLog.ephemeral_volumes:type_name -> common.TouchedVolume - 243, // 115: common.LedgerLogPayload.created_transaction:type_name -> common.CreatedTransaction - 244, // 116: common.LedgerLogPayload.reverted_transaction:type_name -> common.RevertedTransaction - 245, // 117: common.LedgerLogPayload.saved_metadata:type_name -> common.SavedMetadata - 246, // 118: common.LedgerLogPayload.deleted_metadata:type_name -> common.DeletedMetadata - 247, // 119: common.LedgerLogPayload.set_metadata_field_type:type_name -> common.SetMetadataFieldTypeLog - 248, // 120: common.LedgerLogPayload.removed_metadata_field_type:type_name -> common.RemovedMetadataFieldTypeLog - 242, // 121: common.LedgerLogPayload.fill_gap:type_name -> common.FilledGapLog - 240, // 122: common.LedgerLogPayload.create_index:type_name -> common.CreatedIndexLog - 241, // 123: common.LedgerLogPayload.drop_index:type_name -> common.DroppedIndexLog - 289, // 124: common.LedgerLogPayload.added_account_type:type_name -> common.AddedAccountTypeLog - 290, // 125: common.LedgerLogPayload.removed_account_type:type_name -> common.RemovedAccountTypeLog - 291, // 126: common.LedgerLogPayload.updated_default_enforcement_mode:type_name -> common.UpdatedDefaultEnforcementModeLog - 239, // 127: common.LedgerLogPayload.order_skipped:type_name -> common.OrderSkippedLog - 90, // 128: common.FailureDetails.account_address_empty:type_name -> common.AccountAddressEmptyDetails - 91, // 129: common.FailureDetails.account_address_empty_segment:type_name -> common.AccountAddressEmptySegmentDetails - 92, // 130: common.FailureDetails.account_address_invalid_char:type_name -> common.AccountAddressInvalidCharDetails - 93, // 131: common.FailureDetails.account_address_too_long:type_name -> common.AccountAddressTooLongDetails - 94, // 132: common.FailureDetails.account_not_matching_type:type_name -> common.AccountNotMatchingTypeDetails - 95, // 133: common.FailureDetails.account_type_already_exists:type_name -> common.AccountTypeAlreadyExistsDetails - 96, // 134: common.FailureDetails.account_type_conflict:type_name -> common.AccountTypeConflictDetails - 97, // 135: common.FailureDetails.account_type_has_accounts:type_name -> common.AccountTypeHasAccountsDetails - 98, // 136: common.FailureDetails.account_type_name_too_long:type_name -> common.AccountTypeNameTooLongDetails - 99, // 137: common.FailureDetails.account_type_not_found:type_name -> common.AccountTypeNotFoundDetails - 100, // 138: common.FailureDetails.account_type_pattern_too_long:type_name -> common.AccountTypePatternTooLongDetails - 101, // 139: common.FailureDetails.asset_invalid:type_name -> common.AssetInvalidDetails - 102, // 140: common.FailureDetails.balance_not_found:type_name -> common.BalanceNotFoundDetails - 103, // 141: common.FailureDetails.checkpoint_id_required:type_name -> common.CheckpointIdRequiredDetails - 104, // 142: common.FailureDetails.checkpoint_limit_reached:type_name -> common.CheckpointLimitReachedDetails - 105, // 143: common.FailureDetails.checkpoint_not_found:type_name -> common.CheckpointNotFoundDetails - 106, // 144: common.FailureDetails.color_invalid:type_name -> common.ColorInvalidDetails - 107, // 145: common.FailureDetails.color_too_long:type_name -> common.ColorTooLongDetails - 108, // 146: common.FailureDetails.empty_transaction:type_name -> common.EmptyTransactionDetails - 109, // 147: common.FailureDetails.idempotency_key_conflict:type_name -> common.IdempotencyKeyConflictDetails - 110, // 148: common.FailureDetails.id_required:type_name -> common.IdRequiredDetails - 111, // 149: common.FailureDetails.index_already_exists:type_name -> common.IndexAlreadyExistsDetails - 112, // 150: common.FailureDetails.index_not_found:type_name -> common.IndexNotFoundDetails - 113, // 151: common.FailureDetails.index_target_unsupported:type_name -> common.IndexTargetUnsupportedDetails - 114, // 152: common.FailureDetails.insufficient_funds:type_name -> common.InsufficientFundsDetails - 116, // 153: common.FailureDetails.invalid_cron_expression:type_name -> common.InvalidCronExpressionDetails - 117, // 154: common.FailureDetails.invalid_pattern:type_name -> common.InvalidPatternDetails - 118, // 155: common.FailureDetails.invalid_skippable_reason:type_name -> common.InvalidSkippableReasonDetails - 119, // 156: common.FailureDetails.ledger_already_exists:type_name -> common.LedgerAlreadyExistsDetails - 120, // 157: common.FailureDetails.ledger_deleted:type_name -> common.LedgerDeletedDetails - 121, // 158: common.FailureDetails.ledger_in_mirror_mode:type_name -> common.LedgerInMirrorModeDetails - 122, // 159: common.FailureDetails.ledger_name_invalid_char:type_name -> common.LedgerNameInvalidCharDetails - 123, // 160: common.FailureDetails.ledger_name_required:type_name -> common.LedgerNameRequiredDetails - 124, // 161: common.FailureDetails.ledger_name_reserved_prefix:type_name -> common.LedgerNameReservedPrefixDetails - 125, // 162: common.FailureDetails.ledger_name_too_long:type_name -> common.LedgerNameTooLongDetails - 126, // 163: common.FailureDetails.ledger_not_found:type_name -> common.LedgerNotFoundDetails - 127, // 164: common.FailureDetails.ledger_not_in_mirror_mode:type_name -> common.LedgerNotInMirrorModeDetails - 128, // 165: common.FailureDetails.metadata_field_not_in_schema:type_name -> common.MetadataFieldNotInSchemaDetails - 129, // 166: common.FailureDetails.metadata_key_empty:type_name -> common.MetadataKeyEmptyDetails - 131, // 167: common.FailureDetails.metadata_key_invalid_char:type_name -> common.MetadataKeyInvalidCharDetails - 133, // 168: common.FailureDetails.metadata_key_required:type_name -> common.MetadataKeyRequiredDetails - 134, // 169: common.FailureDetails.metadata_limit_exceeded:type_name -> common.MetadataLimitExceededDetails - 138, // 170: common.FailureDetails.metadata_not_found:type_name -> common.MetadataNotFoundDetails - 139, // 171: common.FailureDetails.metadata_value_contains_null_byte:type_name -> common.MetadataValueContainsNullByteDetails - 141, // 172: common.FailureDetails.mirror_http_url_invalid:type_name -> common.MirrorHttpUrlInvalidDetails - 142, // 173: common.FailureDetails.mirror_iam_region_required:type_name -> common.MirrorIamRegionRequiredDetails - 143, // 174: common.FailureDetails.mirror_iam_requires_tls:type_name -> common.MirrorIamRequiresTlsDetails - 144, // 175: common.FailureDetails.mirror_rewrite_rule_invalid:type_name -> common.MirrorRewriteRuleInvalidDetails - 146, // 176: common.FailureDetails.numscript_asset_mismatch:type_name -> common.NumscriptAssetMismatchDetails - 147, // 177: common.FailureDetails.numscript_bad_arity:type_name -> common.NumscriptBadArityDetails - 148, // 178: common.FailureDetails.numscript_bad_meta_value:type_name -> common.NumscriptBadMetaValueDetails - 149, // 179: common.FailureDetails.numscript_bad_portion:type_name -> common.NumscriptBadPortionDetails - 150, // 180: common.FailureDetails.numscript_cannot_cast_scoped_account:type_name -> common.NumscriptCannotCastScopedAccountDetails - 151, // 181: common.FailureDetails.numscript_cannot_cast_to_string:type_name -> common.NumscriptCannotCastToStringDetails - 152, // 182: common.FailureDetails.numscript_cannot_store_scoped_account:type_name -> common.NumscriptCannotStoreScopedAccountDetails - 153, // 183: common.FailureDetails.numscript_compile_bad_arity:type_name -> common.NumscriptCompileBadArityDetails - 154, // 184: common.FailureDetails.numscript_compile_cannot_cast_to_string:type_name -> common.NumscriptCompileCannotCastToStringDetails - 155, // 185: common.FailureDetails.numscript_compile_cannot_store_scoped_account:type_name -> common.NumscriptCompileCannotStoreScopedAccountDetails - 156, // 186: common.FailureDetails.numscript_compile_duplicate_remaining:type_name -> common.NumscriptCompileDuplicateRemainingDetails - 157, // 187: common.FailureDetails.numscript_compile_duplicate_variable:type_name -> common.NumscriptCompileDuplicateVariableDetails - 158, // 188: common.FailureDetails.numscript_compile_experimental_feature:type_name -> common.NumscriptCompileExperimentalFeatureDetails - 159, // 189: common.FailureDetails.numscript_compile_invalid_feature:type_name -> common.NumscriptCompileInvalidFeatureDetails - 160, // 190: common.FailureDetails.numscript_compile_invalid_meta_position:type_name -> common.NumscriptCompileInvalidMetaPositionDetails - 161, // 191: common.FailureDetails.numscript_compile_invalid_type:type_name -> common.NumscriptCompileInvalidTypeDetails - 162, // 192: common.FailureDetails.numscript_compile_invalid_uncapped_source:type_name -> common.NumscriptCompileInvalidUncappedSourceDetails - 163, // 193: common.FailureDetails.numscript_compile_invalid_variable_value:type_name -> common.NumscriptCompileInvalidVariableValueDetails - 164, // 194: common.FailureDetails.numscript_compile_missing_variable:type_name -> common.NumscriptCompileMissingVariableDetails - 165, // 195: common.FailureDetails.numscript_compile_type_mismatch:type_name -> common.NumscriptCompileTypeMismatchDetails - 166, // 196: common.FailureDetails.numscript_compile_unbound_variable:type_name -> common.NumscriptCompileUnboundVariableDetails - 167, // 197: common.FailureDetails.numscript_compile_unknown_function:type_name -> common.NumscriptCompileUnknownFunctionDetails - 168, // 198: common.FailureDetails.numscript_content_required:type_name -> common.NumscriptContentRequiredDetails - 169, // 199: common.FailureDetails.numscript_currency_mismatch:type_name -> common.NumscriptCurrencyMismatchDetails - 170, // 200: common.FailureDetails.numscript_divide_by_zero:type_name -> common.NumscriptDivideByZeroDetails - 172, // 201: common.FailureDetails.numscript_experimental_feature:type_name -> common.NumscriptExperimentalFeatureDetails - 173, // 202: common.FailureDetails.numscript_invalid_account_name:type_name -> common.NumscriptInvalidAccountNameDetails - 174, // 203: common.FailureDetails.numscript_invalid_allotment_in_send_all:type_name -> common.NumscriptInvalidAllotmentInSendAllDetails - 175, // 204: common.FailureDetails.numscript_invalid_allotment_sum:type_name -> common.NumscriptInvalidAllotmentSumDetails - 176, // 205: common.FailureDetails.numscript_invalid_asset:type_name -> common.NumscriptInvalidAssetDetails - 177, // 206: common.FailureDetails.numscript_invalid_color:type_name -> common.NumscriptInvalidColorDetails - 178, // 207: common.FailureDetails.numscript_invalid_feature:type_name -> common.NumscriptInvalidFeatureDetails - 179, // 208: common.FailureDetails.numscript_invalid_monetary_literal:type_name -> common.NumscriptInvalidMonetaryLiteralDetails - 180, // 209: common.FailureDetails.numscript_invalid_nested_meta:type_name -> common.NumscriptInvalidNestedMetaDetails - 181, // 210: common.FailureDetails.numscript_invalid_number_literal:type_name -> common.NumscriptInvalidNumberLiteralDetails - 182, // 211: common.FailureDetails.numscript_invalid_remaining_allotment:type_name -> common.NumscriptInvalidRemainingAllotmentDetails - 183, // 212: common.FailureDetails.numscript_invalid_scope:type_name -> common.NumscriptInvalidScopeDetails - 184, // 213: common.FailureDetails.numscript_invalid_type:type_name -> common.NumscriptInvalidTypeDetails - 185, // 214: common.FailureDetails.numscript_invalid_unbounded_scaling_address:type_name -> common.NumscriptInvalidUnboundedScalingAddressDetails - 186, // 215: common.FailureDetails.numscript_invalid_unbounded_send_all:type_name -> common.NumscriptInvalidUnboundedSendAllDetails - 187, // 216: common.FailureDetails.numscript_invalid_uncapped_source:type_name -> common.NumscriptInvalidUncappedSourceDetails - 188, // 217: common.FailureDetails.numscript_invalid_version:type_name -> common.NumscriptInvalidVersionDetails - 189, // 218: common.FailureDetails.numscript_metadata_not_found:type_name -> common.NumscriptMetadataNotFoundDetails - 191, // 219: common.FailureDetails.numscript_missing_funds:type_name -> common.NumscriptMissingFundsDetails - 192, // 220: common.FailureDetails.numscript_missing_variable:type_name -> common.NumscriptMissingVariableDetails - 193, // 221: common.FailureDetails.numscript_name_invalid_char:type_name -> common.NumscriptNameInvalidCharDetails - 194, // 222: common.FailureDetails.numscript_name_required:type_name -> common.NumscriptNameRequiredDetails - 195, // 223: common.FailureDetails.numscript_name_too_long:type_name -> common.NumscriptNameTooLongDetails - 196, // 224: common.FailureDetails.numscript_negative_amount:type_name -> common.NumscriptNegativeAmountDetails - 197, // 225: common.FailureDetails.numscript_negative_balance:type_name -> common.NumscriptNegativeBalanceDetails - 199, // 226: common.FailureDetails.numscript_negative_portion:type_name -> common.NumscriptNegativePortionDetails - 200, // 227: common.FailureDetails.numscript_not_found:type_name -> common.NumscriptNotFoundDetails - 202, // 228: common.FailureDetails.numscript_parse_error:type_name -> common.NumscriptParseErrorDetails - 203, // 229: common.FailureDetails.numscript_posting_amount_overflow:type_name -> common.NumscriptPostingAmountOverflowDetails - 204, // 230: common.FailureDetails.numscript_posting_negative_amount:type_name -> common.NumscriptPostingNegativeAmountDetails - 205, // 231: common.FailureDetails.numscript_scaling_unsupported:type_name -> common.NumscriptScalingUnsupportedDetails - 206, // 232: common.FailureDetails.numscript_type_error:type_name -> common.NumscriptTypeErrorDetails - 207, // 233: common.FailureDetails.numscript_unbound_function:type_name -> common.NumscriptUnboundFunctionDetails - 208, // 234: common.FailureDetails.numscript_unbound_variable:type_name -> common.NumscriptUnboundVariableDetails - 209, // 235: common.FailureDetails.numscript_version_already_exists:type_name -> common.NumscriptVersionAlreadyExistsDetails - 210, // 236: common.FailureDetails.postings_and_script_conflict:type_name -> common.PostingsAndScriptConflictDetails - 211, // 237: common.FailureDetails.prepared_query_already_exists:type_name -> common.PreparedQueryAlreadyExistsDetails - 212, // 238: common.FailureDetails.prepared_query_name_invalid_char:type_name -> common.PreparedQueryNameInvalidCharDetails - 213, // 239: common.FailureDetails.prepared_query_name_required:type_name -> common.PreparedQueryNameRequiredDetails - 214, // 240: common.FailureDetails.prepared_query_name_too_long:type_name -> common.PreparedQueryNameTooLongDetails - 215, // 241: common.FailureDetails.prepared_query_not_found:type_name -> common.PreparedQueryNotFoundDetails - 216, // 242: common.FailureDetails.prepared_query_required:type_name -> common.PreparedQueryRequiredDetails - 217, // 243: common.FailureDetails.prepared_query_target_unsupported:type_name -> common.PreparedQueryTargetUnsupportedDetails - 218, // 244: common.FailureDetails.revert_target_created_in_batch:type_name -> common.RevertTargetCreatedInBatchDetails - 219, // 245: common.FailureDetails.scoped_balance_unsupported:type_name -> common.ScopedBalanceUnsupportedDetails - 220, // 246: common.FailureDetails.script_and_reference_conflict:type_name -> common.ScriptAndReferenceConflictDetails - 221, // 247: common.FailureDetails.script_required:type_name -> common.ScriptRequiredDetails - 222, // 248: common.FailureDetails.signing_key_id_invalid_char:type_name -> common.SigningKeyIdInvalidCharDetails - 223, // 249: common.FailureDetails.signing_key_id_required:type_name -> common.SigningKeyIdRequiredDetails - 224, // 250: common.FailureDetails.signing_key_id_too_long:type_name -> common.SigningKeyIdTooLongDetails - 225, // 251: common.FailureDetails.signing_key_invalid_length:type_name -> common.SigningKeyInvalidLengthDetails - 226, // 252: common.FailureDetails.sink_already_exists:type_name -> common.SinkAlreadyExistsDetails - 227, // 253: common.FailureDetails.sink_batch_size_too_large:type_name -> common.SinkBatchSizeTooLargeDetails - 228, // 254: common.FailureDetails.sink_controller_mismatch:type_name -> common.SinkControllerMismatchDetails - 229, // 255: common.FailureDetails.sink_not_found:type_name -> common.SinkNotFoundDetails - 230, // 256: common.FailureDetails.target_required:type_name -> common.TargetRequiredDetails - 231, // 257: common.FailureDetails.transaction_already_reverted:type_name -> common.TransactionAlreadyRevertedDetails - 232, // 258: common.FailureDetails.transaction_not_found:type_name -> common.TransactionNotFoundDetails - 233, // 259: common.FailureDetails.transaction_reference_conflict:type_name -> common.TransactionReferenceConflictDetails - 234, // 260: common.FailureDetails.transaction_reference_not_found:type_name -> common.TransactionReferenceNotFoundDetails - 235, // 261: common.FailureDetails.transaction_reference_too_long:type_name -> common.TransactionReferenceTooLongDetails - 236, // 262: common.FailureDetails.transaction_target_missing:type_name -> common.TransactionTargetMissingDetails - 237, // 263: common.FailureDetails.transient_account_non_zero:type_name -> common.TransientAccountNonZeroDetails - 238, // 264: common.FailureDetails.volume_overflow:type_name -> common.VolumeOverflowDetails - 130, // 265: common.FailureDetails.account_metadata_key_empty:type_name -> common.AccountMetadataKeyEmptyDetails - 132, // 266: common.FailureDetails.account_metadata_key_invalid_char:type_name -> common.AccountMetadataKeyInvalidCharDetails - 140, // 267: common.FailureDetails.account_metadata_value_contains_null_byte:type_name -> common.AccountMetadataValueContainsNullByteDetails - 135, // 268: common.FailureDetails.account_metadata_limit_exceeded:type_name -> common.AccountMetadataLimitExceededDetails - 136, // 269: common.FailureDetails.metadata_key_limit_exceeded:type_name -> common.MetadataKeyLimitExceededDetails - 137, // 270: common.FailureDetails.account_metadata_key_limit_exceeded:type_name -> common.AccountMetadataKeyLimitExceededDetails - 115, // 271: common.FailureDetails.numscript_insufficient_funds:type_name -> common.NumscriptInsufficientFundsDetails - 171, // 272: common.FailureDetails.numscript_divide_by_zero_with_numerator:type_name -> common.NumscriptDivideByZeroWithNumeratorDetails - 190, // 273: common.FailureDetails.numscript_scoped_metadata_not_found:type_name -> common.NumscriptScopedMetadataNotFoundDetails - 198, // 274: common.FailureDetails.numscript_scoped_negative_balance:type_name -> common.NumscriptScopedNegativeBalanceDetails - 201, // 275: common.FailureDetails.numscript_version_not_found:type_name -> common.NumscriptVersionNotFoundDetails - 145, // 276: common.FailureDetails.mirror_asset_mapping_invalid:type_name -> common.MirrorAssetMappingInvalidDetails - 89, // 277: common.OrderSkippedLog.details:type_name -> common.FailureDetails - 43, // 278: common.CreatedIndexLog.id:type_name -> common.IndexID - 3, // 279: common.CreatedIndexLog.bound_type:type_name -> common.MetadataType - 43, // 280: common.DroppedIndexLog.id:type_name -> common.IndexID - 26, // 281: common.CreatedTransaction.transaction:type_name -> common.Transaction - 341, // 282: common.CreatedTransaction.account_metadata:type_name -> common.CreatedTransaction.AccountMetadataEntry - 26, // 283: common.RevertedTransaction.revert_transaction:type_name -> common.Transaction - 38, // 284: common.SavedMetadata.target:type_name -> common.Target - 342, // 285: common.SavedMetadata.metadata:type_name -> common.SavedMetadata.MetadataEntry - 38, // 286: common.DeletedMetadata.target:type_name -> common.Target - 2, // 287: common.SetMetadataFieldTypeLog.target_type:type_name -> common.TargetType - 3, // 288: common.SetMetadataFieldTypeLog.type:type_name -> common.MetadataType - 2, // 289: common.RemovedMetadataFieldTypeLog.target_type:type_name -> common.TargetType - 43, // 290: common.RemovedMetadataFieldTypeLog.dropped_index:type_name -> common.IndexID - 270, // 291: common.MirrorSourceConfig.http:type_name -> common.HttpMirrorSourceConfig - 272, // 292: common.MirrorSourceConfig.postgres:type_name -> common.PostgresMirrorSourceConfig - 251, // 293: common.MirrorSourceConfig.rewrite_rules:type_name -> common.MirrorRewriteRule - 250, // 294: common.MirrorSourceConfig.asset_mappings:type_name -> common.MirrorAssetMapping - 252, // 295: common.MirrorRewriteRule.created_transaction:type_name -> common.CreatedTransactionRule - 253, // 296: common.MirrorRewriteRule.reverted_transaction:type_name -> common.RevertedTransactionRule - 254, // 297: common.MirrorRewriteRule.saved_metadata:type_name -> common.SavedMetadataRule - 255, // 298: common.MirrorRewriteRule.deleted_metadata:type_name -> common.DeletedMetadataRule - 256, // 299: common.MirrorRewriteRule.any_variant:type_name -> common.AnyVariantRule - 257, // 300: common.CreatedTransactionRule.actions:type_name -> common.CreatedTransactionAction - 258, // 301: common.RevertedTransactionRule.actions:type_name -> common.RevertedTransactionAction - 259, // 302: common.SavedMetadataRule.actions:type_name -> common.SavedMetadataAction - 260, // 303: common.DeletedMetadataRule.actions:type_name -> common.DeletedMetadataAction - 261, // 304: common.AnyVariantRule.actions:type_name -> common.AnyVariantAction - 262, // 305: common.CreatedTransactionAction.rewrite_address:type_name -> common.RewriteAddressAction - 263, // 306: common.CreatedTransactionAction.set_metadata:type_name -> common.SetMetadataAction - 264, // 307: common.CreatedTransactionAction.delete_metadata:type_name -> common.DeleteMetadataAction - 265, // 308: common.CreatedTransactionAction.set_account_metadata:type_name -> common.SetAccountMetadataAction - 266, // 309: common.CreatedTransactionAction.delete_account_metadata:type_name -> common.DeleteAccountMetadataAction - 267, // 310: common.CreatedTransactionAction.set_account_metadata_from_address:type_name -> common.SetAccountMetadataFromAddressAction - 269, // 311: common.CreatedTransactionAction.drop:type_name -> common.DropAction - 262, // 312: common.RevertedTransactionAction.rewrite_address:type_name -> common.RewriteAddressAction - 263, // 313: common.RevertedTransactionAction.set_metadata:type_name -> common.SetMetadataAction - 264, // 314: common.RevertedTransactionAction.delete_metadata:type_name -> common.DeleteMetadataAction - 269, // 315: common.RevertedTransactionAction.drop:type_name -> common.DropAction - 262, // 316: common.SavedMetadataAction.rewrite_address:type_name -> common.RewriteAddressAction - 263, // 317: common.SavedMetadataAction.set_metadata:type_name -> common.SetMetadataAction - 264, // 318: common.SavedMetadataAction.delete_metadata:type_name -> common.DeleteMetadataAction - 269, // 319: common.SavedMetadataAction.drop:type_name -> common.DropAction - 262, // 320: common.DeletedMetadataAction.rewrite_address:type_name -> common.RewriteAddressAction - 269, // 321: common.DeletedMetadataAction.drop:type_name -> common.DropAction - 262, // 322: common.AnyVariantAction.rewrite_address:type_name -> common.RewriteAddressAction - 269, // 323: common.AnyVariantAction.drop:type_name -> common.DropAction - 268, // 324: common.SetAccountMetadataFromAddressAction.replacements:type_name -> common.SetAccountMetadataFromAddressReplacement - 271, // 325: common.HttpMirrorSourceConfig.oauth2_client_credentials:type_name -> common.OAuth2ClientCredentials - 273, // 326: common.PostgresMirrorSourceConfig.aws_iam_auth:type_name -> common.PostgresAwsIamAuth - 19, // 327: common.MirrorSyncError.occurred_at:type_name -> common.Timestamp - 10, // 328: common.MirrorSyncProgress.state:type_name -> common.MirrorSyncState - 274, // 329: common.MirrorSyncProgress.error:type_name -> common.MirrorSyncError - 19, // 330: common.LedgerInfo.created_at:type_name -> common.Timestamp - 19, // 331: common.LedgerInfo.deleted_at:type_name -> common.Timestamp - 40, // 332: common.LedgerInfo.metadata_schema:type_name -> common.MetadataSchema - 9, // 333: common.LedgerInfo.mode:type_name -> common.LedgerMode - 249, // 334: common.LedgerInfo.mirror_source:type_name -> common.MirrorSourceConfig - 275, // 335: common.LedgerInfo.mirror_sync_progress:type_name -> common.MirrorSyncProgress - 343, // 336: common.LedgerInfo.account_types:type_name -> common.LedgerInfo.AccountTypesEntry - 12, // 337: common.LedgerInfo.default_enforcement_mode:type_name -> common.ChartEnforcementMode - 344, // 338: common.LedgerInfo.metadata:type_name -> common.LedgerInfo.MetadataEntry - 38, // 339: common.SaveMetadataCommand.target:type_name -> common.Target - 345, // 340: common.SaveMetadataCommand.metadata:type_name -> common.SaveMetadataCommand.MetadataEntry - 38, // 341: common.DeleteMetadataCommand.target:type_name -> common.Target - 346, // 342: common.TransactionState.metadata:type_name -> common.TransactionState.MetadataEntry - 19, // 343: common.TransactionState.timestamp:type_name -> common.Timestamp - 25, // 344: common.TransactionState.postings:type_name -> common.Posting - 19, // 345: common.TransactionState.reverted_at:type_name -> common.Timestamp - 281, // 346: common.IdempotencyKeyValue.failure:type_name -> common.IdempotencyFailure - 89, // 347: common.IdempotencyFailure.details:type_name -> common.FailureDetails - 285, // 348: common.SegmentType.uuid:type_name -> common.UUIDConstraint - 286, // 349: common.SegmentType.uint64:type_name -> common.Uint64Constraint - 287, // 350: common.SegmentType.bytes:type_name -> common.BytesConstraint - 13, // 351: common.AccountType.persistence:type_name -> common.AccountTypePersistence - 347, // 352: common.AccountType.segment_types:type_name -> common.AccountType.SegmentTypesEntry - 288, // 353: common.AddedAccountTypeLog.account_type:type_name -> common.AccountType - 12, // 354: common.UpdatedDefaultEnforcementModeLog.enforcement_mode:type_name -> common.ChartEnforcementMode - 305, // 355: common.QueryFilter.field:type_name -> common.FieldCondition - 311, // 356: common.QueryFilter.address:type_name -> common.AddressMatch - 301, // 357: common.QueryFilter.and:type_name -> common.AndFilter - 302, // 358: common.QueryFilter.or:type_name -> common.OrFilter - 303, // 359: common.QueryFilter.not:type_name -> common.NotFilter - 293, // 360: common.QueryFilter.reference:type_name -> common.ReferenceCondition - 298, // 361: common.QueryFilter.builtin_uint:type_name -> common.BuiltinUintCondition - 296, // 362: common.QueryFilter.ledger:type_name -> common.LedgerCondition - 297, // 363: common.QueryFilter.log_id:type_name -> common.LogIdCondition - 299, // 364: common.QueryFilter.log_builtin_uint:type_name -> common.LogBuiltinUintCondition - 300, // 365: common.QueryFilter.account_has_asset:type_name -> common.AccountHasAssetCondition - 294, // 366: common.QueryFilter.reverted:type_name -> common.RevertedCondition - 295, // 367: common.QueryFilter.audit:type_name -> common.AuditCondition - 306, // 368: common.ReferenceCondition.cond:type_name -> common.StringCondition - 14, // 369: common.AuditCondition.field:type_name -> common.AuditField - 306, // 370: common.AuditCondition.string_cond:type_name -> common.StringCondition - 308, // 371: common.AuditCondition.uint_cond:type_name -> common.UintCondition - 306, // 372: common.LedgerCondition.cond:type_name -> common.StringCondition - 308, // 373: common.LogIdCondition.cond:type_name -> common.UintCondition - 4, // 374: common.BuiltinUintCondition.field:type_name -> common.TransactionBuiltinIndex - 308, // 375: common.BuiltinUintCondition.cond:type_name -> common.UintCondition - 6, // 376: common.LogBuiltinUintCondition.field:type_name -> common.LogBuiltinIndex - 308, // 377: common.LogBuiltinUintCondition.cond:type_name -> common.UintCondition - 292, // 378: common.AndFilter.filters:type_name -> common.QueryFilter - 292, // 379: common.OrFilter.filters:type_name -> common.QueryFilter - 292, // 380: common.NotFilter.filter:type_name -> common.QueryFilter - 304, // 381: common.FieldCondition.field:type_name -> common.FieldRef - 306, // 382: common.FieldCondition.string_cond:type_name -> common.StringCondition - 307, // 383: common.FieldCondition.int_cond:type_name -> common.IntCondition - 308, // 384: common.FieldCondition.uint_cond:type_name -> common.UintCondition - 309, // 385: common.FieldCondition.bool_cond:type_name -> common.BoolCondition - 310, // 386: common.FieldCondition.exists_cond:type_name -> common.ExistsCondition - 15, // 387: common.AddressMatch.role:type_name -> common.AddressRole - 292, // 388: common.PreparedQuery.filter:type_name -> common.QueryFilter - 16, // 389: common.PreparedQuery.target:type_name -> common.QueryTarget - 24, // 390: common.AggregatedVolume.input:type_name -> common.Uint256 - 24, // 391: common.AggregatedVolume.output:type_name -> common.Uint256 - 313, // 392: common.AggregateResult.volumes:type_name -> common.AggregatedVolume - 315, // 393: common.AggregateResult.groups:type_name -> common.GroupedAggregateResult - 313, // 394: common.GroupedAggregateResult.volumes:type_name -> common.AggregatedVolume - 36, // 395: common.PreparedQueryCursor.account_data:type_name -> common.Account - 26, // 396: common.PreparedQueryCursor.transaction_data:type_name -> common.Transaction - 46, // 397: common.PreparedQueryCursor.log_data:type_name -> common.Log - 319, // 398: common.AuthenticatedCaller.identity:type_name -> common.CallerIdentity - 320, // 399: common.CallerSnapshot.authenticated:type_name -> common.AuthenticatedCaller - 321, // 400: common.CallerSnapshot.anonymous:type_name -> common.AnonymousCaller - 322, // 401: common.CallerSnapshot.system:type_name -> common.SystemCaller - 323, // 402: common.CallerSnapshot.auth_disabled:type_name -> common.AuthDisabledCaller - 325, // 403: common.BackupStorage.s3:type_name -> common.S3StorageConfig - 326, // 404: common.BackupStorage.azure:type_name -> common.AzureStorageConfig - 328, // 405: common.ListOptions.read:type_name -> common.ReadOptions - 292, // 406: common.ListOptions.filter:type_name -> common.QueryFilter - 21, // 407: common.MetadataMap.ValuesEntry.value:type_name -> common.MetadataValue - 21, // 408: common.Transaction.MetadataEntry.value:type_name -> common.MetadataValue - 32, // 409: common.PostCommitVolumes.VolumesByAccountEntry.value:type_name -> common.VolumesByAssets - 21, // 410: common.Account.MetadataEntry.value:type_name -> common.MetadataValue - 39, // 411: common.MetadataSchema.AccountFieldsEntry.value:type_name -> common.MetadataFieldSchema - 39, // 412: common.MetadataSchema.TransactionFieldsEntry.value:type_name -> common.MetadataFieldSchema - 39, // 413: common.MetadataSchema.LedgerFieldsEntry.value:type_name -> common.MetadataFieldSchema - 21, // 414: common.SavedLedgerMetadataLog.MetadataEntry.value:type_name -> common.MetadataValue - 288, // 415: common.CreatedLedgerLog.AccountTypesEntry.value:type_name -> common.AccountType - 21, // 416: common.CreatedLedgerLog.MetadataEntry.value:type_name -> common.MetadataValue - 22, // 417: common.CreatedTransaction.AccountMetadataEntry.value:type_name -> common.MetadataMap - 21, // 418: common.SavedMetadata.MetadataEntry.value:type_name -> common.MetadataValue - 288, // 419: common.LedgerInfo.AccountTypesEntry.value:type_name -> common.AccountType - 21, // 420: common.LedgerInfo.MetadataEntry.value:type_name -> common.MetadataValue - 21, // 421: common.SaveMetadataCommand.MetadataEntry.value:type_name -> common.MetadataValue - 21, // 422: common.TransactionState.MetadataEntry.value:type_name -> common.MetadataValue - 284, // 423: common.AccountType.SegmentTypesEntry.value:type_name -> common.SegmentType - 349, // 424: common.auth_policy:extendee -> google.protobuf.MethodOptions - 350, // 425: common.ledger_json:extendee -> google.protobuf.MessageOptions - 351, // 426: common.ledger_json_default:extendee -> google.protobuf.FileOptions - 350, // 427: common.ledger_json_manual:extendee -> google.protobuf.MessageOptions - 352, // 428: common.allowed_query_targets:extendee -> google.protobuf.FieldOptions - 352, // 429: common.valid_on_no_query_target:extendee -> google.protobuf.FieldOptions - 352, // 430: common.ledger_log_is_history:extendee -> google.protobuf.FieldOptions - 352, // 431: common.sensitive:extendee -> google.protobuf.FieldOptions - 352, // 432: common.sensitive_url:extendee -> google.protobuf.FieldOptions - 352, // 433: common.failure_reason:extendee -> google.protobuf.FieldOptions - 18, // 434: common.auth_policy:type_name -> common.MethodAuthPolicy - 16, // 435: common.allowed_query_targets:type_name -> common.QueryTarget - 11, // 436: common.failure_reason:type_name -> common.ErrorReason - 437, // [437:437] is the sub-list for method output_type - 437, // [437:437] is the sub-list for method input_type - 434, // [434:437] is the sub-list for extension type_name - 424, // [424:434] is the sub-list for extension extendee - 0, // [0:424] is the sub-list for field type_name + 330, // 81: common.ClusterPolicy.capability_activations:type_name -> common.CapabilityActivation + 312, // 82: common.CreatedPreparedQueryLog.query:type_name -> common.PreparedQuery + 292, // 83: common.UpdatedPreparedQueryLog.previous_filter:type_name -> common.QueryFilter + 292, // 84: common.UpdatedPreparedQueryLog.new_filter:type_name -> common.QueryFilter + 16, // 85: common.UpdatedPreparedQueryLog.previous_target:type_name -> common.QueryTarget + 16, // 86: common.UpdatedPreparedQueryLog.new_target:type_name -> common.QueryTarget + 339, // 87: common.SavedLedgerMetadataLog.metadata:type_name -> common.SavedLedgerMetadataLog.MetadataEntry + 19, // 88: common.NumscriptInfo.created_at:type_name -> common.Timestamp + 66, // 89: common.SavedNumscriptLog.info:type_name -> common.NumscriptInfo + 19, // 90: common.NumscriptVersionEntry.created_at:type_name -> common.Timestamp + 19, // 91: common.TemplateUsage.last_used:type_name -> common.Timestamp + 19, // 92: common.CreatedQueryCheckpointLog.created_at:type_name -> common.Timestamp + 77, // 93: common.SinkConfig.nats:type_name -> common.NatsSinkConfig + 78, // 94: common.SinkConfig.clickhouse:type_name -> common.ClickHouseSinkConfig + 79, // 95: common.SinkConfig.kafka:type_name -> common.KafkaSinkConfig + 80, // 96: common.SinkConfig.http:type_name -> common.HttpSinkConfig + 81, // 97: common.SinkConfig.databricks:type_name -> common.DatabricksSinkConfig + 8, // 98: common.SinkConfig.event_types:type_name -> common.EventType + 76, // 99: common.SinkStatus.error:type_name -> common.SinkError + 19, // 100: common.SinkError.occurred_at:type_name -> common.Timestamp + 82, // 101: common.DatabricksSinkConfig.oauth_m2m:type_name -> common.DatabricksOAuthM2M + 19, // 102: common.CreatedLedgerLog.created_at:type_name -> common.Timestamp + 40, // 103: common.CreatedLedgerLog.metadata_schema:type_name -> common.MetadataSchema + 9, // 104: common.CreatedLedgerLog.mode:type_name -> common.LedgerMode + 249, // 105: common.CreatedLedgerLog.mirror_source:type_name -> common.MirrorSourceConfig + 340, // 106: common.CreatedLedgerLog.account_types:type_name -> common.CreatedLedgerLog.AccountTypesEntry + 12, // 107: common.CreatedLedgerLog.default_enforcement_mode:type_name -> common.ChartEnforcementMode + 341, // 108: common.CreatedLedgerLog.metadata:type_name -> common.CreatedLedgerLog.MetadataEntry + 19, // 109: common.DeletedLedgerLog.deleted_at:type_name -> common.Timestamp + 86, // 110: common.ApplyLedgerLog.log:type_name -> common.LedgerLog + 88, // 111: common.LedgerLog.data:type_name -> common.LedgerLogPayload + 19, // 112: common.LedgerLog.date:type_name -> common.Timestamp + 87, // 113: common.LedgerLog.purged_volumes:type_name -> common.TouchedVolume + 87, // 114: common.LedgerLog.new_kept_volumes:type_name -> common.TouchedVolume + 87, // 115: common.LedgerLog.ephemeral_volumes:type_name -> common.TouchedVolume + 243, // 116: common.LedgerLogPayload.created_transaction:type_name -> common.CreatedTransaction + 244, // 117: common.LedgerLogPayload.reverted_transaction:type_name -> common.RevertedTransaction + 245, // 118: common.LedgerLogPayload.saved_metadata:type_name -> common.SavedMetadata + 246, // 119: common.LedgerLogPayload.deleted_metadata:type_name -> common.DeletedMetadata + 247, // 120: common.LedgerLogPayload.set_metadata_field_type:type_name -> common.SetMetadataFieldTypeLog + 248, // 121: common.LedgerLogPayload.removed_metadata_field_type:type_name -> common.RemovedMetadataFieldTypeLog + 242, // 122: common.LedgerLogPayload.fill_gap:type_name -> common.FilledGapLog + 240, // 123: common.LedgerLogPayload.create_index:type_name -> common.CreatedIndexLog + 241, // 124: common.LedgerLogPayload.drop_index:type_name -> common.DroppedIndexLog + 289, // 125: common.LedgerLogPayload.added_account_type:type_name -> common.AddedAccountTypeLog + 290, // 126: common.LedgerLogPayload.removed_account_type:type_name -> common.RemovedAccountTypeLog + 291, // 127: common.LedgerLogPayload.updated_default_enforcement_mode:type_name -> common.UpdatedDefaultEnforcementModeLog + 239, // 128: common.LedgerLogPayload.order_skipped:type_name -> common.OrderSkippedLog + 90, // 129: common.FailureDetails.account_address_empty:type_name -> common.AccountAddressEmptyDetails + 91, // 130: common.FailureDetails.account_address_empty_segment:type_name -> common.AccountAddressEmptySegmentDetails + 92, // 131: common.FailureDetails.account_address_invalid_char:type_name -> common.AccountAddressInvalidCharDetails + 93, // 132: common.FailureDetails.account_address_too_long:type_name -> common.AccountAddressTooLongDetails + 94, // 133: common.FailureDetails.account_not_matching_type:type_name -> common.AccountNotMatchingTypeDetails + 95, // 134: common.FailureDetails.account_type_already_exists:type_name -> common.AccountTypeAlreadyExistsDetails + 96, // 135: common.FailureDetails.account_type_conflict:type_name -> common.AccountTypeConflictDetails + 97, // 136: common.FailureDetails.account_type_has_accounts:type_name -> common.AccountTypeHasAccountsDetails + 98, // 137: common.FailureDetails.account_type_name_too_long:type_name -> common.AccountTypeNameTooLongDetails + 99, // 138: common.FailureDetails.account_type_not_found:type_name -> common.AccountTypeNotFoundDetails + 100, // 139: common.FailureDetails.account_type_pattern_too_long:type_name -> common.AccountTypePatternTooLongDetails + 101, // 140: common.FailureDetails.asset_invalid:type_name -> common.AssetInvalidDetails + 102, // 141: common.FailureDetails.balance_not_found:type_name -> common.BalanceNotFoundDetails + 103, // 142: common.FailureDetails.checkpoint_id_required:type_name -> common.CheckpointIdRequiredDetails + 104, // 143: common.FailureDetails.checkpoint_limit_reached:type_name -> common.CheckpointLimitReachedDetails + 105, // 144: common.FailureDetails.checkpoint_not_found:type_name -> common.CheckpointNotFoundDetails + 106, // 145: common.FailureDetails.color_invalid:type_name -> common.ColorInvalidDetails + 107, // 146: common.FailureDetails.color_too_long:type_name -> common.ColorTooLongDetails + 108, // 147: common.FailureDetails.empty_transaction:type_name -> common.EmptyTransactionDetails + 109, // 148: common.FailureDetails.idempotency_key_conflict:type_name -> common.IdempotencyKeyConflictDetails + 110, // 149: common.FailureDetails.id_required:type_name -> common.IdRequiredDetails + 111, // 150: common.FailureDetails.index_already_exists:type_name -> common.IndexAlreadyExistsDetails + 112, // 151: common.FailureDetails.index_not_found:type_name -> common.IndexNotFoundDetails + 113, // 152: common.FailureDetails.index_target_unsupported:type_name -> common.IndexTargetUnsupportedDetails + 114, // 153: common.FailureDetails.insufficient_funds:type_name -> common.InsufficientFundsDetails + 116, // 154: common.FailureDetails.invalid_cron_expression:type_name -> common.InvalidCronExpressionDetails + 117, // 155: common.FailureDetails.invalid_pattern:type_name -> common.InvalidPatternDetails + 118, // 156: common.FailureDetails.invalid_skippable_reason:type_name -> common.InvalidSkippableReasonDetails + 119, // 157: common.FailureDetails.ledger_already_exists:type_name -> common.LedgerAlreadyExistsDetails + 120, // 158: common.FailureDetails.ledger_deleted:type_name -> common.LedgerDeletedDetails + 121, // 159: common.FailureDetails.ledger_in_mirror_mode:type_name -> common.LedgerInMirrorModeDetails + 122, // 160: common.FailureDetails.ledger_name_invalid_char:type_name -> common.LedgerNameInvalidCharDetails + 123, // 161: common.FailureDetails.ledger_name_required:type_name -> common.LedgerNameRequiredDetails + 124, // 162: common.FailureDetails.ledger_name_reserved_prefix:type_name -> common.LedgerNameReservedPrefixDetails + 125, // 163: common.FailureDetails.ledger_name_too_long:type_name -> common.LedgerNameTooLongDetails + 126, // 164: common.FailureDetails.ledger_not_found:type_name -> common.LedgerNotFoundDetails + 127, // 165: common.FailureDetails.ledger_not_in_mirror_mode:type_name -> common.LedgerNotInMirrorModeDetails + 128, // 166: common.FailureDetails.metadata_field_not_in_schema:type_name -> common.MetadataFieldNotInSchemaDetails + 129, // 167: common.FailureDetails.metadata_key_empty:type_name -> common.MetadataKeyEmptyDetails + 131, // 168: common.FailureDetails.metadata_key_invalid_char:type_name -> common.MetadataKeyInvalidCharDetails + 133, // 169: common.FailureDetails.metadata_key_required:type_name -> common.MetadataKeyRequiredDetails + 134, // 170: common.FailureDetails.metadata_limit_exceeded:type_name -> common.MetadataLimitExceededDetails + 138, // 171: common.FailureDetails.metadata_not_found:type_name -> common.MetadataNotFoundDetails + 139, // 172: common.FailureDetails.metadata_value_contains_null_byte:type_name -> common.MetadataValueContainsNullByteDetails + 141, // 173: common.FailureDetails.mirror_http_url_invalid:type_name -> common.MirrorHttpUrlInvalidDetails + 142, // 174: common.FailureDetails.mirror_iam_region_required:type_name -> common.MirrorIamRegionRequiredDetails + 143, // 175: common.FailureDetails.mirror_iam_requires_tls:type_name -> common.MirrorIamRequiresTlsDetails + 144, // 176: common.FailureDetails.mirror_rewrite_rule_invalid:type_name -> common.MirrorRewriteRuleInvalidDetails + 146, // 177: common.FailureDetails.numscript_asset_mismatch:type_name -> common.NumscriptAssetMismatchDetails + 147, // 178: common.FailureDetails.numscript_bad_arity:type_name -> common.NumscriptBadArityDetails + 148, // 179: common.FailureDetails.numscript_bad_meta_value:type_name -> common.NumscriptBadMetaValueDetails + 149, // 180: common.FailureDetails.numscript_bad_portion:type_name -> common.NumscriptBadPortionDetails + 150, // 181: common.FailureDetails.numscript_cannot_cast_scoped_account:type_name -> common.NumscriptCannotCastScopedAccountDetails + 151, // 182: common.FailureDetails.numscript_cannot_cast_to_string:type_name -> common.NumscriptCannotCastToStringDetails + 152, // 183: common.FailureDetails.numscript_cannot_store_scoped_account:type_name -> common.NumscriptCannotStoreScopedAccountDetails + 153, // 184: common.FailureDetails.numscript_compile_bad_arity:type_name -> common.NumscriptCompileBadArityDetails + 154, // 185: common.FailureDetails.numscript_compile_cannot_cast_to_string:type_name -> common.NumscriptCompileCannotCastToStringDetails + 155, // 186: common.FailureDetails.numscript_compile_cannot_store_scoped_account:type_name -> common.NumscriptCompileCannotStoreScopedAccountDetails + 156, // 187: common.FailureDetails.numscript_compile_duplicate_remaining:type_name -> common.NumscriptCompileDuplicateRemainingDetails + 157, // 188: common.FailureDetails.numscript_compile_duplicate_variable:type_name -> common.NumscriptCompileDuplicateVariableDetails + 158, // 189: common.FailureDetails.numscript_compile_experimental_feature:type_name -> common.NumscriptCompileExperimentalFeatureDetails + 159, // 190: common.FailureDetails.numscript_compile_invalid_feature:type_name -> common.NumscriptCompileInvalidFeatureDetails + 160, // 191: common.FailureDetails.numscript_compile_invalid_meta_position:type_name -> common.NumscriptCompileInvalidMetaPositionDetails + 161, // 192: common.FailureDetails.numscript_compile_invalid_type:type_name -> common.NumscriptCompileInvalidTypeDetails + 162, // 193: common.FailureDetails.numscript_compile_invalid_uncapped_source:type_name -> common.NumscriptCompileInvalidUncappedSourceDetails + 163, // 194: common.FailureDetails.numscript_compile_invalid_variable_value:type_name -> common.NumscriptCompileInvalidVariableValueDetails + 164, // 195: common.FailureDetails.numscript_compile_missing_variable:type_name -> common.NumscriptCompileMissingVariableDetails + 165, // 196: common.FailureDetails.numscript_compile_type_mismatch:type_name -> common.NumscriptCompileTypeMismatchDetails + 166, // 197: common.FailureDetails.numscript_compile_unbound_variable:type_name -> common.NumscriptCompileUnboundVariableDetails + 167, // 198: common.FailureDetails.numscript_compile_unknown_function:type_name -> common.NumscriptCompileUnknownFunctionDetails + 168, // 199: common.FailureDetails.numscript_content_required:type_name -> common.NumscriptContentRequiredDetails + 169, // 200: common.FailureDetails.numscript_currency_mismatch:type_name -> common.NumscriptCurrencyMismatchDetails + 170, // 201: common.FailureDetails.numscript_divide_by_zero:type_name -> common.NumscriptDivideByZeroDetails + 172, // 202: common.FailureDetails.numscript_experimental_feature:type_name -> common.NumscriptExperimentalFeatureDetails + 173, // 203: common.FailureDetails.numscript_invalid_account_name:type_name -> common.NumscriptInvalidAccountNameDetails + 174, // 204: common.FailureDetails.numscript_invalid_allotment_in_send_all:type_name -> common.NumscriptInvalidAllotmentInSendAllDetails + 175, // 205: common.FailureDetails.numscript_invalid_allotment_sum:type_name -> common.NumscriptInvalidAllotmentSumDetails + 176, // 206: common.FailureDetails.numscript_invalid_asset:type_name -> common.NumscriptInvalidAssetDetails + 177, // 207: common.FailureDetails.numscript_invalid_color:type_name -> common.NumscriptInvalidColorDetails + 178, // 208: common.FailureDetails.numscript_invalid_feature:type_name -> common.NumscriptInvalidFeatureDetails + 179, // 209: common.FailureDetails.numscript_invalid_monetary_literal:type_name -> common.NumscriptInvalidMonetaryLiteralDetails + 180, // 210: common.FailureDetails.numscript_invalid_nested_meta:type_name -> common.NumscriptInvalidNestedMetaDetails + 181, // 211: common.FailureDetails.numscript_invalid_number_literal:type_name -> common.NumscriptInvalidNumberLiteralDetails + 182, // 212: common.FailureDetails.numscript_invalid_remaining_allotment:type_name -> common.NumscriptInvalidRemainingAllotmentDetails + 183, // 213: common.FailureDetails.numscript_invalid_scope:type_name -> common.NumscriptInvalidScopeDetails + 184, // 214: common.FailureDetails.numscript_invalid_type:type_name -> common.NumscriptInvalidTypeDetails + 185, // 215: common.FailureDetails.numscript_invalid_unbounded_scaling_address:type_name -> common.NumscriptInvalidUnboundedScalingAddressDetails + 186, // 216: common.FailureDetails.numscript_invalid_unbounded_send_all:type_name -> common.NumscriptInvalidUnboundedSendAllDetails + 187, // 217: common.FailureDetails.numscript_invalid_uncapped_source:type_name -> common.NumscriptInvalidUncappedSourceDetails + 188, // 218: common.FailureDetails.numscript_invalid_version:type_name -> common.NumscriptInvalidVersionDetails + 189, // 219: common.FailureDetails.numscript_metadata_not_found:type_name -> common.NumscriptMetadataNotFoundDetails + 191, // 220: common.FailureDetails.numscript_missing_funds:type_name -> common.NumscriptMissingFundsDetails + 192, // 221: common.FailureDetails.numscript_missing_variable:type_name -> common.NumscriptMissingVariableDetails + 193, // 222: common.FailureDetails.numscript_name_invalid_char:type_name -> common.NumscriptNameInvalidCharDetails + 194, // 223: common.FailureDetails.numscript_name_required:type_name -> common.NumscriptNameRequiredDetails + 195, // 224: common.FailureDetails.numscript_name_too_long:type_name -> common.NumscriptNameTooLongDetails + 196, // 225: common.FailureDetails.numscript_negative_amount:type_name -> common.NumscriptNegativeAmountDetails + 197, // 226: common.FailureDetails.numscript_negative_balance:type_name -> common.NumscriptNegativeBalanceDetails + 199, // 227: common.FailureDetails.numscript_negative_portion:type_name -> common.NumscriptNegativePortionDetails + 200, // 228: common.FailureDetails.numscript_not_found:type_name -> common.NumscriptNotFoundDetails + 202, // 229: common.FailureDetails.numscript_parse_error:type_name -> common.NumscriptParseErrorDetails + 203, // 230: common.FailureDetails.numscript_posting_amount_overflow:type_name -> common.NumscriptPostingAmountOverflowDetails + 204, // 231: common.FailureDetails.numscript_posting_negative_amount:type_name -> common.NumscriptPostingNegativeAmountDetails + 205, // 232: common.FailureDetails.numscript_scaling_unsupported:type_name -> common.NumscriptScalingUnsupportedDetails + 206, // 233: common.FailureDetails.numscript_type_error:type_name -> common.NumscriptTypeErrorDetails + 207, // 234: common.FailureDetails.numscript_unbound_function:type_name -> common.NumscriptUnboundFunctionDetails + 208, // 235: common.FailureDetails.numscript_unbound_variable:type_name -> common.NumscriptUnboundVariableDetails + 209, // 236: common.FailureDetails.numscript_version_already_exists:type_name -> common.NumscriptVersionAlreadyExistsDetails + 210, // 237: common.FailureDetails.postings_and_script_conflict:type_name -> common.PostingsAndScriptConflictDetails + 211, // 238: common.FailureDetails.prepared_query_already_exists:type_name -> common.PreparedQueryAlreadyExistsDetails + 212, // 239: common.FailureDetails.prepared_query_name_invalid_char:type_name -> common.PreparedQueryNameInvalidCharDetails + 213, // 240: common.FailureDetails.prepared_query_name_required:type_name -> common.PreparedQueryNameRequiredDetails + 214, // 241: common.FailureDetails.prepared_query_name_too_long:type_name -> common.PreparedQueryNameTooLongDetails + 215, // 242: common.FailureDetails.prepared_query_not_found:type_name -> common.PreparedQueryNotFoundDetails + 216, // 243: common.FailureDetails.prepared_query_required:type_name -> common.PreparedQueryRequiredDetails + 217, // 244: common.FailureDetails.prepared_query_target_unsupported:type_name -> common.PreparedQueryTargetUnsupportedDetails + 218, // 245: common.FailureDetails.revert_target_created_in_batch:type_name -> common.RevertTargetCreatedInBatchDetails + 219, // 246: common.FailureDetails.scoped_balance_unsupported:type_name -> common.ScopedBalanceUnsupportedDetails + 220, // 247: common.FailureDetails.script_and_reference_conflict:type_name -> common.ScriptAndReferenceConflictDetails + 221, // 248: common.FailureDetails.script_required:type_name -> common.ScriptRequiredDetails + 222, // 249: common.FailureDetails.signing_key_id_invalid_char:type_name -> common.SigningKeyIdInvalidCharDetails + 223, // 250: common.FailureDetails.signing_key_id_required:type_name -> common.SigningKeyIdRequiredDetails + 224, // 251: common.FailureDetails.signing_key_id_too_long:type_name -> common.SigningKeyIdTooLongDetails + 225, // 252: common.FailureDetails.signing_key_invalid_length:type_name -> common.SigningKeyInvalidLengthDetails + 226, // 253: common.FailureDetails.sink_already_exists:type_name -> common.SinkAlreadyExistsDetails + 227, // 254: common.FailureDetails.sink_batch_size_too_large:type_name -> common.SinkBatchSizeTooLargeDetails + 228, // 255: common.FailureDetails.sink_controller_mismatch:type_name -> common.SinkControllerMismatchDetails + 229, // 256: common.FailureDetails.sink_not_found:type_name -> common.SinkNotFoundDetails + 230, // 257: common.FailureDetails.target_required:type_name -> common.TargetRequiredDetails + 231, // 258: common.FailureDetails.transaction_already_reverted:type_name -> common.TransactionAlreadyRevertedDetails + 232, // 259: common.FailureDetails.transaction_not_found:type_name -> common.TransactionNotFoundDetails + 233, // 260: common.FailureDetails.transaction_reference_conflict:type_name -> common.TransactionReferenceConflictDetails + 234, // 261: common.FailureDetails.transaction_reference_not_found:type_name -> common.TransactionReferenceNotFoundDetails + 235, // 262: common.FailureDetails.transaction_reference_too_long:type_name -> common.TransactionReferenceTooLongDetails + 236, // 263: common.FailureDetails.transaction_target_missing:type_name -> common.TransactionTargetMissingDetails + 237, // 264: common.FailureDetails.transient_account_non_zero:type_name -> common.TransientAccountNonZeroDetails + 238, // 265: common.FailureDetails.volume_overflow:type_name -> common.VolumeOverflowDetails + 130, // 266: common.FailureDetails.account_metadata_key_empty:type_name -> common.AccountMetadataKeyEmptyDetails + 132, // 267: common.FailureDetails.account_metadata_key_invalid_char:type_name -> common.AccountMetadataKeyInvalidCharDetails + 140, // 268: common.FailureDetails.account_metadata_value_contains_null_byte:type_name -> common.AccountMetadataValueContainsNullByteDetails + 135, // 269: common.FailureDetails.account_metadata_limit_exceeded:type_name -> common.AccountMetadataLimitExceededDetails + 136, // 270: common.FailureDetails.metadata_key_limit_exceeded:type_name -> common.MetadataKeyLimitExceededDetails + 137, // 271: common.FailureDetails.account_metadata_key_limit_exceeded:type_name -> common.AccountMetadataKeyLimitExceededDetails + 115, // 272: common.FailureDetails.numscript_insufficient_funds:type_name -> common.NumscriptInsufficientFundsDetails + 171, // 273: common.FailureDetails.numscript_divide_by_zero_with_numerator:type_name -> common.NumscriptDivideByZeroWithNumeratorDetails + 190, // 274: common.FailureDetails.numscript_scoped_metadata_not_found:type_name -> common.NumscriptScopedMetadataNotFoundDetails + 198, // 275: common.FailureDetails.numscript_scoped_negative_balance:type_name -> common.NumscriptScopedNegativeBalanceDetails + 201, // 276: common.FailureDetails.numscript_version_not_found:type_name -> common.NumscriptVersionNotFoundDetails + 145, // 277: common.FailureDetails.mirror_asset_mapping_invalid:type_name -> common.MirrorAssetMappingInvalidDetails + 89, // 278: common.OrderSkippedLog.details:type_name -> common.FailureDetails + 43, // 279: common.CreatedIndexLog.id:type_name -> common.IndexID + 3, // 280: common.CreatedIndexLog.bound_type:type_name -> common.MetadataType + 43, // 281: common.DroppedIndexLog.id:type_name -> common.IndexID + 26, // 282: common.CreatedTransaction.transaction:type_name -> common.Transaction + 342, // 283: common.CreatedTransaction.account_metadata:type_name -> common.CreatedTransaction.AccountMetadataEntry + 26, // 284: common.RevertedTransaction.revert_transaction:type_name -> common.Transaction + 38, // 285: common.SavedMetadata.target:type_name -> common.Target + 343, // 286: common.SavedMetadata.metadata:type_name -> common.SavedMetadata.MetadataEntry + 38, // 287: common.DeletedMetadata.target:type_name -> common.Target + 2, // 288: common.SetMetadataFieldTypeLog.target_type:type_name -> common.TargetType + 3, // 289: common.SetMetadataFieldTypeLog.type:type_name -> common.MetadataType + 2, // 290: common.RemovedMetadataFieldTypeLog.target_type:type_name -> common.TargetType + 43, // 291: common.RemovedMetadataFieldTypeLog.dropped_index:type_name -> common.IndexID + 270, // 292: common.MirrorSourceConfig.http:type_name -> common.HttpMirrorSourceConfig + 272, // 293: common.MirrorSourceConfig.postgres:type_name -> common.PostgresMirrorSourceConfig + 251, // 294: common.MirrorSourceConfig.rewrite_rules:type_name -> common.MirrorRewriteRule + 250, // 295: common.MirrorSourceConfig.asset_mappings:type_name -> common.MirrorAssetMapping + 252, // 296: common.MirrorRewriteRule.created_transaction:type_name -> common.CreatedTransactionRule + 253, // 297: common.MirrorRewriteRule.reverted_transaction:type_name -> common.RevertedTransactionRule + 254, // 298: common.MirrorRewriteRule.saved_metadata:type_name -> common.SavedMetadataRule + 255, // 299: common.MirrorRewriteRule.deleted_metadata:type_name -> common.DeletedMetadataRule + 256, // 300: common.MirrorRewriteRule.any_variant:type_name -> common.AnyVariantRule + 257, // 301: common.CreatedTransactionRule.actions:type_name -> common.CreatedTransactionAction + 258, // 302: common.RevertedTransactionRule.actions:type_name -> common.RevertedTransactionAction + 259, // 303: common.SavedMetadataRule.actions:type_name -> common.SavedMetadataAction + 260, // 304: common.DeletedMetadataRule.actions:type_name -> common.DeletedMetadataAction + 261, // 305: common.AnyVariantRule.actions:type_name -> common.AnyVariantAction + 262, // 306: common.CreatedTransactionAction.rewrite_address:type_name -> common.RewriteAddressAction + 263, // 307: common.CreatedTransactionAction.set_metadata:type_name -> common.SetMetadataAction + 264, // 308: common.CreatedTransactionAction.delete_metadata:type_name -> common.DeleteMetadataAction + 265, // 309: common.CreatedTransactionAction.set_account_metadata:type_name -> common.SetAccountMetadataAction + 266, // 310: common.CreatedTransactionAction.delete_account_metadata:type_name -> common.DeleteAccountMetadataAction + 267, // 311: common.CreatedTransactionAction.set_account_metadata_from_address:type_name -> common.SetAccountMetadataFromAddressAction + 269, // 312: common.CreatedTransactionAction.drop:type_name -> common.DropAction + 262, // 313: common.RevertedTransactionAction.rewrite_address:type_name -> common.RewriteAddressAction + 263, // 314: common.RevertedTransactionAction.set_metadata:type_name -> common.SetMetadataAction + 264, // 315: common.RevertedTransactionAction.delete_metadata:type_name -> common.DeleteMetadataAction + 269, // 316: common.RevertedTransactionAction.drop:type_name -> common.DropAction + 262, // 317: common.SavedMetadataAction.rewrite_address:type_name -> common.RewriteAddressAction + 263, // 318: common.SavedMetadataAction.set_metadata:type_name -> common.SetMetadataAction + 264, // 319: common.SavedMetadataAction.delete_metadata:type_name -> common.DeleteMetadataAction + 269, // 320: common.SavedMetadataAction.drop:type_name -> common.DropAction + 262, // 321: common.DeletedMetadataAction.rewrite_address:type_name -> common.RewriteAddressAction + 269, // 322: common.DeletedMetadataAction.drop:type_name -> common.DropAction + 262, // 323: common.AnyVariantAction.rewrite_address:type_name -> common.RewriteAddressAction + 269, // 324: common.AnyVariantAction.drop:type_name -> common.DropAction + 268, // 325: common.SetAccountMetadataFromAddressAction.replacements:type_name -> common.SetAccountMetadataFromAddressReplacement + 271, // 326: common.HttpMirrorSourceConfig.oauth2_client_credentials:type_name -> common.OAuth2ClientCredentials + 273, // 327: common.PostgresMirrorSourceConfig.aws_iam_auth:type_name -> common.PostgresAwsIamAuth + 19, // 328: common.MirrorSyncError.occurred_at:type_name -> common.Timestamp + 10, // 329: common.MirrorSyncProgress.state:type_name -> common.MirrorSyncState + 274, // 330: common.MirrorSyncProgress.error:type_name -> common.MirrorSyncError + 19, // 331: common.LedgerInfo.created_at:type_name -> common.Timestamp + 19, // 332: common.LedgerInfo.deleted_at:type_name -> common.Timestamp + 40, // 333: common.LedgerInfo.metadata_schema:type_name -> common.MetadataSchema + 9, // 334: common.LedgerInfo.mode:type_name -> common.LedgerMode + 249, // 335: common.LedgerInfo.mirror_source:type_name -> common.MirrorSourceConfig + 275, // 336: common.LedgerInfo.mirror_sync_progress:type_name -> common.MirrorSyncProgress + 344, // 337: common.LedgerInfo.account_types:type_name -> common.LedgerInfo.AccountTypesEntry + 12, // 338: common.LedgerInfo.default_enforcement_mode:type_name -> common.ChartEnforcementMode + 345, // 339: common.LedgerInfo.metadata:type_name -> common.LedgerInfo.MetadataEntry + 38, // 340: common.SaveMetadataCommand.target:type_name -> common.Target + 346, // 341: common.SaveMetadataCommand.metadata:type_name -> common.SaveMetadataCommand.MetadataEntry + 38, // 342: common.DeleteMetadataCommand.target:type_name -> common.Target + 347, // 343: common.TransactionState.metadata:type_name -> common.TransactionState.MetadataEntry + 19, // 344: common.TransactionState.timestamp:type_name -> common.Timestamp + 25, // 345: common.TransactionState.postings:type_name -> common.Posting + 19, // 346: common.TransactionState.reverted_at:type_name -> common.Timestamp + 281, // 347: common.IdempotencyKeyValue.failure:type_name -> common.IdempotencyFailure + 89, // 348: common.IdempotencyFailure.details:type_name -> common.FailureDetails + 285, // 349: common.SegmentType.uuid:type_name -> common.UUIDConstraint + 286, // 350: common.SegmentType.uint64:type_name -> common.Uint64Constraint + 287, // 351: common.SegmentType.bytes:type_name -> common.BytesConstraint + 13, // 352: common.AccountType.persistence:type_name -> common.AccountTypePersistence + 348, // 353: common.AccountType.segment_types:type_name -> common.AccountType.SegmentTypesEntry + 288, // 354: common.AddedAccountTypeLog.account_type:type_name -> common.AccountType + 12, // 355: common.UpdatedDefaultEnforcementModeLog.enforcement_mode:type_name -> common.ChartEnforcementMode + 305, // 356: common.QueryFilter.field:type_name -> common.FieldCondition + 311, // 357: common.QueryFilter.address:type_name -> common.AddressMatch + 301, // 358: common.QueryFilter.and:type_name -> common.AndFilter + 302, // 359: common.QueryFilter.or:type_name -> common.OrFilter + 303, // 360: common.QueryFilter.not:type_name -> common.NotFilter + 293, // 361: common.QueryFilter.reference:type_name -> common.ReferenceCondition + 298, // 362: common.QueryFilter.builtin_uint:type_name -> common.BuiltinUintCondition + 296, // 363: common.QueryFilter.ledger:type_name -> common.LedgerCondition + 297, // 364: common.QueryFilter.log_id:type_name -> common.LogIdCondition + 299, // 365: common.QueryFilter.log_builtin_uint:type_name -> common.LogBuiltinUintCondition + 300, // 366: common.QueryFilter.account_has_asset:type_name -> common.AccountHasAssetCondition + 294, // 367: common.QueryFilter.reverted:type_name -> common.RevertedCondition + 295, // 368: common.QueryFilter.audit:type_name -> common.AuditCondition + 306, // 369: common.ReferenceCondition.cond:type_name -> common.StringCondition + 14, // 370: common.AuditCondition.field:type_name -> common.AuditField + 306, // 371: common.AuditCondition.string_cond:type_name -> common.StringCondition + 308, // 372: common.AuditCondition.uint_cond:type_name -> common.UintCondition + 306, // 373: common.LedgerCondition.cond:type_name -> common.StringCondition + 308, // 374: common.LogIdCondition.cond:type_name -> common.UintCondition + 4, // 375: common.BuiltinUintCondition.field:type_name -> common.TransactionBuiltinIndex + 308, // 376: common.BuiltinUintCondition.cond:type_name -> common.UintCondition + 6, // 377: common.LogBuiltinUintCondition.field:type_name -> common.LogBuiltinIndex + 308, // 378: common.LogBuiltinUintCondition.cond:type_name -> common.UintCondition + 292, // 379: common.AndFilter.filters:type_name -> common.QueryFilter + 292, // 380: common.OrFilter.filters:type_name -> common.QueryFilter + 292, // 381: common.NotFilter.filter:type_name -> common.QueryFilter + 304, // 382: common.FieldCondition.field:type_name -> common.FieldRef + 306, // 383: common.FieldCondition.string_cond:type_name -> common.StringCondition + 307, // 384: common.FieldCondition.int_cond:type_name -> common.IntCondition + 308, // 385: common.FieldCondition.uint_cond:type_name -> common.UintCondition + 309, // 386: common.FieldCondition.bool_cond:type_name -> common.BoolCondition + 310, // 387: common.FieldCondition.exists_cond:type_name -> common.ExistsCondition + 15, // 388: common.AddressMatch.role:type_name -> common.AddressRole + 292, // 389: common.PreparedQuery.filter:type_name -> common.QueryFilter + 16, // 390: common.PreparedQuery.target:type_name -> common.QueryTarget + 24, // 391: common.AggregatedVolume.input:type_name -> common.Uint256 + 24, // 392: common.AggregatedVolume.output:type_name -> common.Uint256 + 313, // 393: common.AggregateResult.volumes:type_name -> common.AggregatedVolume + 315, // 394: common.AggregateResult.groups:type_name -> common.GroupedAggregateResult + 313, // 395: common.GroupedAggregateResult.volumes:type_name -> common.AggregatedVolume + 36, // 396: common.PreparedQueryCursor.account_data:type_name -> common.Account + 26, // 397: common.PreparedQueryCursor.transaction_data:type_name -> common.Transaction + 46, // 398: common.PreparedQueryCursor.log_data:type_name -> common.Log + 319, // 399: common.AuthenticatedCaller.identity:type_name -> common.CallerIdentity + 320, // 400: common.CallerSnapshot.authenticated:type_name -> common.AuthenticatedCaller + 321, // 401: common.CallerSnapshot.anonymous:type_name -> common.AnonymousCaller + 322, // 402: common.CallerSnapshot.system:type_name -> common.SystemCaller + 323, // 403: common.CallerSnapshot.auth_disabled:type_name -> common.AuthDisabledCaller + 325, // 404: common.BackupStorage.s3:type_name -> common.S3StorageConfig + 326, // 405: common.BackupStorage.azure:type_name -> common.AzureStorageConfig + 328, // 406: common.ListOptions.read:type_name -> common.ReadOptions + 292, // 407: common.ListOptions.filter:type_name -> common.QueryFilter + 21, // 408: common.MetadataMap.ValuesEntry.value:type_name -> common.MetadataValue + 21, // 409: common.Transaction.MetadataEntry.value:type_name -> common.MetadataValue + 32, // 410: common.PostCommitVolumes.VolumesByAccountEntry.value:type_name -> common.VolumesByAssets + 21, // 411: common.Account.MetadataEntry.value:type_name -> common.MetadataValue + 39, // 412: common.MetadataSchema.AccountFieldsEntry.value:type_name -> common.MetadataFieldSchema + 39, // 413: common.MetadataSchema.TransactionFieldsEntry.value:type_name -> common.MetadataFieldSchema + 39, // 414: common.MetadataSchema.LedgerFieldsEntry.value:type_name -> common.MetadataFieldSchema + 21, // 415: common.SavedLedgerMetadataLog.MetadataEntry.value:type_name -> common.MetadataValue + 288, // 416: common.CreatedLedgerLog.AccountTypesEntry.value:type_name -> common.AccountType + 21, // 417: common.CreatedLedgerLog.MetadataEntry.value:type_name -> common.MetadataValue + 22, // 418: common.CreatedTransaction.AccountMetadataEntry.value:type_name -> common.MetadataMap + 21, // 419: common.SavedMetadata.MetadataEntry.value:type_name -> common.MetadataValue + 288, // 420: common.LedgerInfo.AccountTypesEntry.value:type_name -> common.AccountType + 21, // 421: common.LedgerInfo.MetadataEntry.value:type_name -> common.MetadataValue + 21, // 422: common.SaveMetadataCommand.MetadataEntry.value:type_name -> common.MetadataValue + 21, // 423: common.TransactionState.MetadataEntry.value:type_name -> common.MetadataValue + 284, // 424: common.AccountType.SegmentTypesEntry.value:type_name -> common.SegmentType + 350, // 425: common.auth_policy:extendee -> google.protobuf.MethodOptions + 351, // 426: common.ledger_json:extendee -> google.protobuf.MessageOptions + 352, // 427: common.ledger_json_default:extendee -> google.protobuf.FileOptions + 351, // 428: common.ledger_json_manual:extendee -> google.protobuf.MessageOptions + 353, // 429: common.allowed_query_targets:extendee -> google.protobuf.FieldOptions + 353, // 430: common.valid_on_no_query_target:extendee -> google.protobuf.FieldOptions + 353, // 431: common.ledger_log_is_history:extendee -> google.protobuf.FieldOptions + 353, // 432: common.sensitive:extendee -> google.protobuf.FieldOptions + 353, // 433: common.sensitive_url:extendee -> google.protobuf.FieldOptions + 353, // 434: common.failure_reason:extendee -> google.protobuf.FieldOptions + 18, // 435: common.auth_policy:type_name -> common.MethodAuthPolicy + 16, // 436: common.allowed_query_targets:type_name -> common.QueryTarget + 11, // 437: common.failure_reason:type_name -> common.ErrorReason + 438, // [438:438] is the sub-list for method output_type + 438, // [438:438] is the sub-list for method input_type + 435, // [435:438] is the sub-list for extension type_name + 425, // [425:435] is the sub-list for extension extendee + 0, // [0:425] is the sub-list for field type_name } func init() { file_common_proto_init() } @@ -25253,7 +25322,7 @@ func file_common_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_common_proto_rawDesc), len(file_common_proto_rawDesc)), NumEnums: 18, - NumMessages: 330, + NumMessages: 331, NumExtensions: 10, NumServices: 0, }, diff --git a/internal/proto/commonpb/common_dethash.pb.go b/internal/proto/commonpb/common_dethash.pb.go index 608b704f1e..1ded8fcee8 100644 --- a/internal/proto/commonpb/common_dethash.pb.go +++ b/internal/proto/commonpb/common_dethash.pb.go @@ -4962,3 +4962,14 @@ func (m *ListOptions) MarshalDeterministicVT(dAtA []byte) []byte { } return append(dAtA, b...) } + +func (m *CapabilityActivation) MarshalDeterministicVT(dAtA []byte) []byte { + if m == nil { + return dAtA + } + b, err := m.MarshalVT() + if err != nil { + panic("MarshalDeterministicVT: " + err.Error()) + } + return append(dAtA, b...) +} diff --git a/internal/proto/commonpb/common_ledger_json.pb.go b/internal/proto/commonpb/common_ledger_json.pb.go index 90063f4507..e56d38b865 100644 --- a/internal/proto/commonpb/common_ledger_json.pb.go +++ b/internal/proto/commonpb/common_ledger_json.pb.go @@ -323,6 +323,7 @@ func (x *ClusterPolicy) MarshalJSON() ([]byte, error) { MetadataMaxValueBytes uint64 `json:"metadataMaxValueBytes"` MetadataMaxEntityBytes uint64 `json:"metadataMaxEntityBytes"` MetadataMaxCommandBytes uint64 `json:"metadataMaxCommandBytes"` + CapabilityActivations any `json:"capabilityActivations"` }{ Revision: x.Revision, IdempotencyTtlMicros: x.IdempotencyTtlMicros, @@ -332,6 +333,12 @@ func (x *ClusterPolicy) MarshalJSON() ([]byte, error) { MetadataMaxValueBytes: x.MetadataMaxValueBytes, MetadataMaxEntityBytes: x.MetadataMaxEntityBytes, MetadataMaxCommandBytes: x.MetadataMaxCommandBytes, + CapabilityActivations: func() any { + if x.CapabilityActivations == nil { + return []any{} + } + return x.CapabilityActivations + }(), }) } diff --git a/internal/proto/commonpb/common_reader.pb.go b/internal/proto/commonpb/common_reader.pb.go index 130514c3e5..e78ae7e47e 100644 --- a/internal/proto/commonpb/common_reader.pb.go +++ b/internal/proto/commonpb/common_reader.pb.go @@ -3522,6 +3522,7 @@ type ClusterPolicyReader interface { GetMetadataMaxValueBytes() uint64 GetMetadataMaxEntityBytes() uint64 GetMetadataMaxCommandBytes() uint64 + GetCapabilityActivations() CapabilityActivationListReader Mutate() *ClusterPolicy } @@ -3559,6 +3560,10 @@ func (r *clusterPolicyReadonly) GetMetadataMaxCommandBytes() uint64 { return (*ClusterPolicy)(r).GetMetadataMaxCommandBytes() } +func (r *clusterPolicyReadonly) GetCapabilityActivations() CapabilityActivationListReader { + return NewCapabilityActivationListReader((*ClusterPolicy)(r).GetCapabilityActivations()) +} + func (r *clusterPolicyReadonly) Mutate() *ClusterPolicy { return (*ClusterPolicy)(r).CloneVT() } @@ -23051,3 +23056,75 @@ func (l listOptionsListReadonly) Range(yield func(int, ListOptionsReader) bool) func NewListOptionsListReader(s []*ListOptions) ListOptionsListReader { return listOptionsListReadonly(s) } + +// CapabilityActivationReader provides read-only access to CapabilityActivation. +// Call Mutate() to obtain a mutable clone. +type CapabilityActivationReader interface { + GetId() uint32 + GetActive() bool + Mutate() *CapabilityActivation +} + +type capabilityActivationReadonly CapabilityActivation + +func (r *capabilityActivationReadonly) GetId() uint32 { + return (*CapabilityActivation)(r).GetId() +} + +func (r *capabilityActivationReadonly) GetActive() bool { + return (*CapabilityActivation)(r).GetActive() +} + +func (r *capabilityActivationReadonly) Mutate() *CapabilityActivation { + return (*CapabilityActivation)(r).CloneVT() +} + +// AsReader returns a read-only view of this CapabilityActivation. +func (m *CapabilityActivation) AsReader() CapabilityActivationReader { + if m == nil { + return nil + } + return (*capabilityActivationReadonly)(m) +} + +// Mutate returns a mutable deep clone of this CapabilityActivation. +func (m *CapabilityActivation) Mutate() *CapabilityActivation { + return m.CloneVT() +} + +// CapabilityActivationListReader provides read-only iteration over []*CapabilityActivation. +type CapabilityActivationListReader interface { + Len() int + Get(i int) CapabilityActivationReader + Range(yield func(int, CapabilityActivationReader) bool) +} + +type capabilityActivationListReadonly []*CapabilityActivation + +func (l capabilityActivationListReadonly) Len() int { return len(l) } + +func (l capabilityActivationListReadonly) Get(i int) CapabilityActivationReader { + v := l[i] + if v == nil { + return nil + } + return v.AsReader() +} + +func (l capabilityActivationListReadonly) Range(yield func(int, CapabilityActivationReader) bool) { + for i, v := range l { + var r CapabilityActivationReader + if v != nil { + r = v.AsReader() + } + if !yield(i, r) { + return + } + } +} + +// NewCapabilityActivationListReader wraps s for read-only iteration. The returned +// view aliases the underlying slice; do not mutate s afterwards. +func NewCapabilityActivationListReader(s []*CapabilityActivation) CapabilityActivationListReader { + return capabilityActivationListReadonly(s) +} diff --git a/internal/proto/commonpb/common_vtproto.pb.go b/internal/proto/commonpb/common_vtproto.pb.go index 94fb2aeb76..97e8ab6af9 100644 --- a/internal/proto/commonpb/common_vtproto.pb.go +++ b/internal/proto/commonpb/common_vtproto.pb.go @@ -1273,6 +1273,13 @@ func (m *ClusterPolicy) CloneVT() *ClusterPolicy { r.MetadataMaxValueBytes = m.MetadataMaxValueBytes r.MetadataMaxEntityBytes = m.MetadataMaxEntityBytes r.MetadataMaxCommandBytes = m.MetadataMaxCommandBytes + if rhs := m.CapabilityActivations; rhs != nil { + tmpContainer := make([]*CapabilityActivation, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v.CloneVT() + } + r.CapabilityActivations = tmpContainer + } if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -8968,6 +8975,24 @@ func (m *ListOptions) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *CapabilityActivation) CloneVT() *CapabilityActivation { + if m == nil { + return (*CapabilityActivation)(nil) + } + r := new(CapabilityActivation) + r.Id = m.Id + r.Active = m.Active + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *CapabilityActivation) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (this *MethodAuthPolicy) EqualVT(that *MethodAuthPolicy) bool { if this == that { return true @@ -11084,6 +11109,23 @@ func (this *ClusterPolicy) EqualVT(that *ClusterPolicy) bool { if this.MetadataMaxCommandBytes != that.MetadataMaxCommandBytes { return false } + if len(this.CapabilityActivations) != len(that.CapabilityActivations) { + return false + } + for i, vx := range this.CapabilityActivations { + vy := that.CapabilityActivations[i] + if p, q := vx, vy; p != q { + if p == nil { + p = &CapabilityActivation{} + } + if q == nil { + q = &CapabilityActivation{} + } + if !p.EqualVT(q) { + return false + } + } + } return string(this.unknownFields) == string(that.unknownFields) } @@ -23300,6 +23342,28 @@ func (this *ListOptions) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *CapabilityActivation) EqualVT(that *CapabilityActivation) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.Id != that.Id { + return false + } + if this.Active != that.Active { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *CapabilityActivation) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*CapabilityActivation) + if !ok { + return false + } + return this.EqualVT(that) +} func (m *MethodAuthPolicy) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -26376,6 +26440,18 @@ func (m *ClusterPolicy) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.CapabilityActivations) > 0 { + for iNdEx := len(m.CapabilityActivations) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.CapabilityActivations[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x6a + } + } if m.MetadataMaxCommandBytes != 0 { i -= 8 binary.LittleEndian.PutUint64(dAtA[i:], uint64(m.MetadataMaxCommandBytes)) @@ -43586,6 +43662,55 @@ func (m *ListOptions) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *CapabilityActivation) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CapabilityActivation) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *CapabilityActivation) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Active { + i-- + if m.Active { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x10 + } + if m.Id != 0 { + i -= 4 + binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Id)) + i-- + dAtA[i] = 0xd + } + return len(dAtA) - i, nil +} + func (m *MethodAuthPolicy) SizeVT() (n int) { if m == nil { return 0 @@ -44939,6 +45064,12 @@ func (m *ClusterPolicy) SizeVT() (n int) { if m.MetadataMaxCommandBytes != 0 { n += 9 } + if len(m.CapabilityActivations) > 0 { + for _, e := range m.CapabilityActivations { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } n += len(m.unknownFields) return n } @@ -52304,6 +52435,22 @@ func (m *ListOptions) SizeVT() (n int) { return n } +func (m *CapabilityActivation) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Id != 0 { + n += 5 + } + if m.Active { + n += 2 + } + n += len(m.unknownFields) + return n +} + func (m *MethodAuthPolicy) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -59594,6 +59741,40 @@ func (m *ClusterPolicy) UnmarshalVT(dAtA []byte) error { } m.MetadataMaxCommandBytes = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 + case 13: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CapabilityActivations", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CapabilityActivations = append(m.CapabilityActivations, &CapabilityActivation{}) + if err := m.CapabilityActivations[len(m.CapabilityActivations)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -96618,3 +96799,84 @@ func (m *ListOptions) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *CapabilityActivation) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CapabilityActivation: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CapabilityActivation: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 5 { + return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) + } + m.Id = 0 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + m.Id = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Active", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Active = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/internal/proto/commonpb/ledger_info_json_test.go b/internal/proto/commonpb/ledger_info_json_test.go index c1892bf0e1..41be037204 100644 --- a/internal/proto/commonpb/ledger_info_json_test.go +++ b/internal/proto/commonpb/ledger_info_json_test.go @@ -94,5 +94,5 @@ func TestSetClusterPolicyLogJSONProjectsNestedPolicy(t *testing.T) { MetadataMaxCommandBytes: 16_384, }}) require.NoError(t, err) - require.JSONEq(t, `{"policy":{"revision":7,"idempotencyTtlMicros":86400000000,"queryCheckpointLimit":0,"metadataMaxEntriesPerEntity":0,"metadataMaxKeyBytes":256,"metadataMaxValueBytes":0,"metadataMaxEntityBytes":0,"metadataMaxCommandBytes":16384}}`, string(data)) + require.JSONEq(t, `{"policy":{"revision":7,"idempotencyTtlMicros":86400000000,"queryCheckpointLimit":0,"metadataMaxEntriesPerEntity":0,"metadataMaxKeyBytes":256,"metadataMaxValueBytes":0,"metadataMaxEntityBytes":0,"metadataMaxCommandBytes":16384,"capabilityActivations":[]}}`, string(data)) } diff --git a/internal/proto/raftcmdpb/raft_cmd.pb.go b/internal/proto/raftcmdpb/raft_cmd.pb.go index 89ff76ef20..12ab706338 100644 --- a/internal/proto/raftcmdpb/raft_cmd.pb.go +++ b/internal/proto/raftcmdpb/raft_cmd.pb.go @@ -3434,6 +3434,9 @@ type Proposal struct { // // Deprecated: Marked as deprecated in raft_cmd.proto. TechnicalUpdates []*TechnicalUpdate `protobuf:"bytes,9,rep,name=technical_updates,json=technicalUpdates,proto3" json:"technical_updates,omitempty"` + // Stable capability IDs required by the final serialized proposal. This is + // checked after any plan rebuild and before the bytes enter Raft. + RequiredCapabilities []uint32 `protobuf:"fixed32,10,rep,packed,name=required_capabilities,json=requiredCapabilities,proto3" json:"required_capabilities,omitempty"` // Exclusive with legacy fields 2 and 9. Reading this envelope is supported; // emitting it on Raft requires a separately validated cluster-wide cutover. // @@ -3541,6 +3544,13 @@ func (x *Proposal) GetTechnicalUpdates() []*TechnicalUpdate { return nil } +func (x *Proposal) GetRequiredCapabilities() []uint32 { + if x != nil { + return x.RequiredCapabilities + } + return nil +} + func (x *Proposal) GetPayload() isProposal_Payload { if x != nil { return x.Payload @@ -5571,8 +5581,12 @@ type PeerAddress struct { RaftAddress string `protobuf:"bytes,2,opt,name=raft_address,json=raftAddress,proto3" json:"raft_address,omitempty"` ServiceAddress string `protobuf:"bytes,3,opt,name=service_address,json=serviceAddress,proto3" json:"service_address,omitempty"` InstanceId []byte `protobuf:"bytes,4,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Capability evidence is bound to this concrete (node_id, instance_id) + // incarnation. It is persisted with the peer row so a restart cannot + // accidentally reuse evidence from an older WAL/PVC incarnation. + Capabilities []uint32 `protobuf:"fixed32,5,rep,packed,name=capabilities,proto3" json:"capabilities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PeerAddress) Reset() { @@ -5633,6 +5647,13 @@ func (x *PeerAddress) GetInstanceId() []byte { return nil } +func (x *PeerAddress) GetCapabilities() []uint32 { + if x != nil { + return x.Capabilities + } + return nil +} + // RemovedMemberEntry is the persisted blacklist row keyed by // [ZoneClusterPersistent][SubGlobRemovedMembers][node_id BE 8][instance_id 16]. Written // by both the consensus and force removal paths, read by JoinAsLearner @@ -5991,7 +6012,7 @@ const file_raft_cmd_proto_rawDesc = "" + "\x06orders\x18\x01 \x03(\v2\v.raft.OrderR\x06orders\x12B\n" + "\x11technical_updates\x18\x02 \x03(\v2\x15.raft.TechnicalUpdateR\x10technicalUpdates\"I\n" + "\x18SetClusterPolicyProposal\x12-\n" + - "\x06policy\x18\x01 \x01(\v2\x15.common.ClusterPolicyR\x06policy\"\xd3\x04\n" + + "\x06policy\x18\x01 \x01(\v2\x15.common.ClusterPolicyR\x06policy\"\x88\x05\n" + "\bProposal\x12\x0e\n" + "\x02id\x18\x01 \x01(\x06R\x02id\x12'\n" + "\x06orders\x18\x02 \x03(\v2\v.raft.OrderB\x02\x18\x01R\x06orders\x12%\n" + @@ -6001,7 +6022,9 @@ const file_raft_cmd_proto_rawDesc = "" + "\x0fcaller_snapshot\x18\x06 \x01(\v2\x16.common.CallerSnapshotR\x0ecallerSnapshot\x125\n" + "\vidempotency\x18\a \x01(\v2\x13.common.IdempotencyR\vidempotency\x129\n" + "\tsignature\x18\b \x01(\v2\x1b.signature.SignedApplyBatchR\tsignature\x12F\n" + - "\x11technical_updates\x18\t \x03(\v2\x15.raft.TechnicalUpdateB\x02\x18\x01R\x10technicalUpdates\x12,\n" + + "\x11technical_updates\x18\t \x03(\v2\x15.raft.TechnicalUpdateB\x02\x18\x01R\x10technicalUpdates\x123\n" + + "\x15required_capabilities\x18\n" + + " \x03(\aR\x14requiredCapabilities\x12,\n" + "\x05batch\x18\v \x01(\v2\x14.raft.ExecutionBatchH\x00R\x05batch\x12N\n" + "\x12set_cluster_policy\x18\f \x01(\v2\x1e.raft.SetClusterPolicyProposalH\x00R\x10setClusterPolicyB\t\n" + "\apayload\"\xe8\x04\n" + @@ -6137,13 +6160,14 @@ const file_raft_cmd_proto_rawDesc = "" + "\n" + "base_index\x18\x01 \x01(\x06R\tbaseIndex\"B\n" + "\x11CacheSnapshotMeta\x12-\n" + - "\x12current_generation\x18\x01 \x01(\x06R\x11currentGeneration\"\x93\x01\n" + + "\x12current_generation\x18\x01 \x01(\x06R\x11currentGeneration\"\xb7\x01\n" + "\vPeerAddress\x12\x17\n" + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12!\n" + "\fraft_address\x18\x02 \x01(\tR\vraftAddress\x12'\n" + "\x0fservice_address\x18\x03 \x01(\tR\x0eserviceAddress\x12\x1f\n" + "\vinstance_id\x18\x04 \x01(\fR\n" + - "instanceId\"\x85\x01\n" + + "instanceId\x12\"\n" + + "\fcapabilities\x18\x05 \x03(\aR\fcapabilities\"\x85\x01\n" + "\x12RemovedMemberEntry\x12\x17\n" + "\anode_id\x18\x01 \x01(\x04R\x06nodeId\x12\x1f\n" + "\vinstance_id\x18\x02 \x01(\fR\n" + diff --git a/internal/proto/raftcmdpb/raft_cmd_dethash.pb.go b/internal/proto/raftcmdpb/raft_cmd_dethash.pb.go index faf50cee54..76071a867e 100644 --- a/internal/proto/raftcmdpb/raft_cmd_dethash.pb.go +++ b/internal/proto/raftcmdpb/raft_cmd_dethash.pb.go @@ -1565,6 +1565,15 @@ func (m *Proposal) MarshalToSizedBufferDeterministicVT(dAtA []byte) (int, error) i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.RequiredCapabilities) > 0 { + i -= 4 * len(m.RequiredCapabilities) + for iNdEx, v := range m.RequiredCapabilities { + binary.LittleEndian.PutUint32(dAtA[i+iNdEx*4:], uint32(v)) + } + i = protohelpers.EncodeVarint(dAtA, i, uint64(4*len(m.RequiredCapabilities))) + i-- + dAtA[i] = 0x52 + } if len(m.TechnicalUpdates) > 0 { for iNdEx := len(m.TechnicalUpdates) - 1; iNdEx >= 0; iNdEx-- { size, _ := m.TechnicalUpdates[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) diff --git a/internal/proto/raftcmdpb/raft_cmd_reader.pb.go b/internal/proto/raftcmdpb/raft_cmd_reader.pb.go index 2245bd304b..89841e5195 100644 --- a/internal/proto/raftcmdpb/raft_cmd_reader.pb.go +++ b/internal/proto/raftcmdpb/raft_cmd_reader.pb.go @@ -3872,6 +3872,7 @@ type ProposalReader interface { GetIdempotency() commonpb.IdempotencyReader GetSignature() signaturepb.SignedApplyBatchReader GetTechnicalUpdates() TechnicalUpdateListReader + GetRequiredCapabilities() []uint32 GetPayload() isProposal_Payload Mutate() *Proposal } @@ -3934,6 +3935,10 @@ func (r *proposalReadonly) GetTechnicalUpdates() TechnicalUpdateListReader { return NewTechnicalUpdateListReader((*Proposal)(r).GetTechnicalUpdates()) } +func (r *proposalReadonly) GetRequiredCapabilities() []uint32 { + return slices.Clone((*Proposal)(r).GetRequiredCapabilities()) +} + func (r *proposalReadonly) GetPayload() isProposal_Payload { return (*Proposal)(r).GetPayload() } @@ -5960,6 +5965,7 @@ type PeerAddressReader interface { GetRaftAddress() string GetServiceAddress() string GetInstanceId() []byte + GetCapabilities() []uint32 Mutate() *PeerAddress } @@ -5981,6 +5987,10 @@ func (r *peerAddressReadonly) GetInstanceId() []byte { return bytes.Clone((*PeerAddress)(r).GetInstanceId()) } +func (r *peerAddressReadonly) GetCapabilities() []uint32 { + return slices.Clone((*PeerAddress)(r).GetCapabilities()) +} + func (r *peerAddressReadonly) Mutate() *PeerAddress { return (*PeerAddress)(r).CloneVT() } diff --git a/internal/proto/raftcmdpb/raft_cmd_vtproto.pb.go b/internal/proto/raftcmdpb/raft_cmd_vtproto.pb.go index 732a54601f..0579b2fe96 100644 --- a/internal/proto/raftcmdpb/raft_cmd_vtproto.pb.go +++ b/internal/proto/raftcmdpb/raft_cmd_vtproto.pb.go @@ -1409,6 +1409,11 @@ func (m *Proposal) CloneVT() *Proposal { } r.TechnicalUpdates = tmpContainer } + if rhs := m.RequiredCapabilities; rhs != nil { + tmpContainer := make([]uint32, len(rhs)) + copy(tmpContainer, rhs) + r.RequiredCapabilities = tmpContainer + } if m.Payload != nil { r.Payload = m.Payload.(interface{ CloneVT() isProposal_Payload }).CloneVT() } @@ -2168,6 +2173,11 @@ func (m *PeerAddress) CloneVT() *PeerAddress { copy(tmpBytes, rhs) r.InstanceId = tmpBytes } + if rhs := m.Capabilities; rhs != nil { + tmpContainer := make([]uint32, len(rhs)) + copy(tmpContainer, rhs) + r.Capabilities = tmpContainer + } if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -4674,6 +4684,15 @@ func (this *Proposal) EqualVT(that *Proposal) bool { } } } + if len(this.RequiredCapabilities) != len(that.RequiredCapabilities) { + return false + } + for i, vx := range this.RequiredCapabilities { + vy := that.RequiredCapabilities[i] + if vx != vy { + return false + } + } return string(this.unknownFields) == string(that.unknownFields) } @@ -5931,6 +5950,15 @@ func (this *PeerAddress) EqualVT(that *PeerAddress) bool { if string(this.InstanceId) != string(that.InstanceId) { return false } + if len(this.Capabilities) != len(that.Capabilities) { + return false + } + for i, vx := range this.Capabilities { + vy := that.Capabilities[i] + if vx != vy { + return false + } + } return string(this.unknownFields) == string(that.unknownFields) } @@ -9320,6 +9348,15 @@ func (m *Proposal) MarshalToSizedBufferVT(dAtA []byte) (int, error) { } i -= size } + if len(m.RequiredCapabilities) > 0 { + for iNdEx := len(m.RequiredCapabilities) - 1; iNdEx >= 0; iNdEx-- { + i -= 4 + binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.RequiredCapabilities[iNdEx])) + } + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RequiredCapabilities)*4)) + i-- + dAtA[i] = 0x52 + } if len(m.TechnicalUpdates) > 0 { for iNdEx := len(m.TechnicalUpdates) - 1; iNdEx >= 0; iNdEx-- { size, err := m.TechnicalUpdates[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) @@ -11210,6 +11247,15 @@ func (m *PeerAddress) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Capabilities) > 0 { + for iNdEx := len(m.Capabilities) - 1; iNdEx >= 0; iNdEx-- { + i -= 4 + binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Capabilities[iNdEx])) + } + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Capabilities)*4)) + i-- + dAtA[i] = 0x2a + } if len(m.InstanceId) > 0 { i -= len(m.InstanceId) copy(dAtA[i:], m.InstanceId) @@ -11411,12 +11457,14 @@ func (m *Proposal) ResetVT() { mm.ResetVT() } f1 := m.TechnicalUpdates[:0] + f2 := m.RequiredCapabilities[:0] if oneof, ok := m.Payload.(*Proposal_Batch); ok { oneof.Batch.ReturnToVTPool() } m.Reset() m.Orders = f0 m.TechnicalUpdates = f1 + m.RequiredCapabilities = f2 } } func (m *Proposal) ReturnToVTPool() { @@ -13023,6 +13071,9 @@ func (m *Proposal) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if len(m.RequiredCapabilities) > 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(len(m.RequiredCapabilities)*4)) + len(m.RequiredCapabilities)*4 + } if vtmsg, ok := m.Payload.(interface{ SizeVT() int }); ok { n += vtmsg.SizeVT() } @@ -13822,6 +13873,9 @@ func (m *PeerAddress) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if len(m.Capabilities) > 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(len(m.Capabilities)*4)) + len(m.Capabilities)*4 + } n += len(m.unknownFields) return n } @@ -22116,6 +22170,58 @@ func (m *Proposal) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 10: + if wireType == 5 { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.RequiredCapabilities = append(m.RequiredCapabilities, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + elementCount = packedLen / 4 + if elementCount != 0 && len(m.RequiredCapabilities) == 0 && cap(m.RequiredCapabilities) < elementCount { + m.RequiredCapabilities = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.RequiredCapabilities = append(m.RequiredCapabilities, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RequiredCapabilities", wireType) + } case 11: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Batch", wireType) @@ -26175,6 +26281,58 @@ func (m *PeerAddress) UnmarshalVT(dAtA []byte) error { m.InstanceId = []byte{} } iNdEx = postIndex + case 5: + if wireType == 5 { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + elementCount = packedLen / 4 + if elementCount != 0 && len(m.Capabilities) == 0 { + m.Capabilities = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.Capabilities = append(m.Capabilities, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Capabilities", wireType) + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/misc/proto/cluster.proto b/misc/proto/cluster.proto index c651e49505..29a4b8029d 100644 --- a/misc/proto/cluster.proto +++ b/misc/proto/cluster.proto @@ -287,6 +287,7 @@ message AddLearnerRequest { string raft_address = 2; // Raft transport address of the learner string service_address = 3; // Service API address of the learner bytes instance_id = 4; // 16-byte UUID from the learner's persisted INSTANCE_ID marker; required + repeated fixed32 capabilities = 5; // Capabilities supported by the learner binary } message AddLearnerResponse {} diff --git a/misc/proto/cluster_bootstrap.proto b/misc/proto/cluster_bootstrap.proto index 030ef7ad65..b0c1a32ef3 100644 --- a/misc/proto/cluster_bootstrap.proto +++ b/misc/proto/cluster_bootstrap.proto @@ -8,7 +8,7 @@ option go_package = "github.com/formancehq/ledger/v3/internal/proto/clusterboots // // Unlike ClusterService (exposed on the external ServiceServer behind the // user JWT/scope pipeline), this service lives on the internal RaftServer -// alongside the Raft transport. It carries the two RPCs required by a node +// alongside the Raft transport. It carries the RPCs required by a node // that is joining an existing cluster: // // 1. GetPeers — discover the current set of voter nodes and their @@ -16,6 +16,8 @@ option go_package = "github.com/formancehq/ledger/v3/internal/proto/clusterboots // snapshot before starting Raft. // 2. JoinAsLearner — register with the leader as a new learner, which // proposes the corresponding ConfChange. +// 3. RefreshMembership — refresh the committed endpoints and capability +// evidence of an existing member after restart. // // Authentication is the RaftServer's own: cluster-id metadata (always) and // the cluster-secret bearer when configured. No user JWT or scope check @@ -23,6 +25,7 @@ option go_package = "github.com/formancehq/ledger/v3/internal/proto/clusterboots service ClusterBootstrapService { rpc GetPeers(GetPeersRequest) returns (GetPeersResponse); rpc JoinAsLearner(JoinAsLearnerRequest) returns (JoinAsLearnerResponse); + rpc RefreshMembership(RefreshMembershipRequest) returns (RefreshMembershipResponse); } message GetPeersRequest {} @@ -38,6 +41,7 @@ message PeerInfo { string raft_address = 2; string service_address = 3; bytes instance_id = 4; // 16-byte UUID identifying this peer's WAL/PVC incarnation; required + repeated fixed32 capabilities = 5; // capabilities supported by this binary } message JoinAsLearnerRequest { @@ -45,6 +49,21 @@ message JoinAsLearnerRequest { string raft_address = 2; // Raft transport address of the learner string service_address = 3; // Service API address of the learner bytes instance_id = 4; // 16-byte UUID identifying this peer's WAL/PVC incarnation (EN-1045); required + repeated fixed32 capabilities = 5; // capabilities supported by this binary } message JoinAsLearnerResponse {} + +// RefreshMembership carries the current identity, endpoints and capability +// evidence of a member that already exists in the committed Raft membership. +// It is used after a normal restart, where the node must not re-run the +// learner-join path. +message RefreshMembershipRequest { + uint64 node_id = 1; + string raft_address = 2; + string service_address = 3; + bytes instance_id = 4; + repeated fixed32 capabilities = 5; +} + +message RefreshMembershipResponse {} diff --git a/misc/proto/common.proto b/misc/proto/common.proto index 857f4ee2c3..f2166b6270 100644 --- a/misc/proto/common.proto +++ b/misc/proto/common.proto @@ -552,6 +552,8 @@ message ClusterPolicy { fixed64 metadata_max_value_bytes = 6; fixed64 metadata_max_entity_bytes = 7; fixed64 metadata_max_command_bytes = 8; + // Capability activation is replicated policy, never a node-local flag. + repeated CapabilityActivation capability_activations = 13; } // CreatedPreparedQueryLog records the creation of a prepared query. @@ -2866,3 +2868,10 @@ message ListOptions { // rejects unsupported variants with InvalidArgument. QueryFilter filter = 5; } + +// CapabilityActivation is intentionally small and wire-stable. IDs are fixed +// integers owned by the producer, not hashes of implementation names. +message CapabilityActivation { + fixed32 id = 1; + bool active = 2; +} diff --git a/misc/proto/raft_cmd.proto b/misc/proto/raft_cmd.proto index 4bfc079382..8cec3f05a6 100644 --- a/misc/proto/raft_cmd.proto +++ b/misc/proto/raft_cmd.proto @@ -451,6 +451,10 @@ message Proposal { // Order.coverage_bits. repeated TechnicalUpdate technical_updates = 9 [deprecated = true]; + // Stable capability IDs required by the final serialized proposal. This is + // checked after any plan rebuild and before the bytes enter Raft. + repeated fixed32 required_capabilities = 10; + // Exclusive with legacy fields 2 and 9. Reading this envelope is supported; // emitting it on Raft requires a separately validated cluster-wide cutover. oneof payload { @@ -835,6 +839,10 @@ message PeerAddress { string raft_address = 2; string service_address = 3; bytes instance_id = 4; + // Capability evidence is bound to this concrete (node_id, instance_id) + // incarnation. It is persisted with the peer row so a restart cannot + // accidentally reuse evidence from an older WAL/PVC incarnation. + repeated fixed32 capabilities = 5; } // RemovedMemberEntry is the persisted blacklist row keyed by diff --git a/pkg/testserver/gateway.go b/pkg/testserver/gateway.go index 55ccadd0b9..00a7448c64 100644 --- a/pkg/testserver/gateway.go +++ b/pkg/testserver/gateway.go @@ -485,6 +485,10 @@ func (g *clusterBootstrapServiceGateway) JoinAsLearner(ctx context.Context, req return g.client.JoinAsLearner(forwardIncomingMetadata(ctx), req) } +func (g *clusterBootstrapServiceGateway) RefreshMembership(ctx context.Context, req *clusterbootstrappb.RefreshMembershipRequest) (*clusterbootstrappb.RefreshMembershipResponse, error) { + return g.client.RefreshMembership(forwardIncomingMetadata(ctx), req) +} + func forwardIncomingMetadata(ctx context.Context) context.Context { md, _ := metadata.FromIncomingContext(ctx) diff --git a/scripts/agent-check-pr b/scripts/agent-check-pr index bff0426ef1..0fe460e55d 100755 --- a/scripts/agent-check-pr +++ b/scripts/agent-check-pr @@ -79,6 +79,7 @@ needs_schemathesis=false needs_operator=false needs_protocol=false needs_proto_lint=false +needs_capability_contract=false has_change=false # Gate selection runs under the host bash, before the Nix re-exec below; macOS # ships bash 3.2, where "${arr[@]}" on an empty array is an unbound-variable @@ -102,12 +103,27 @@ classify_path() { # Service contract or revision edits are classified against the base by # scripts/checkprotocolversion, independently of the gates below. case "$path" in + misc/proto/common.proto|misc/proto/raft_cmd.proto) + needs_protocol=true + needs_capability_contract=true + ;; misc/proto/*|pkg/grpcprotocol/*) needs_protocol=true ;; esac case "$path" in misc/proto/*|tools/buf-plugin-ledger/*) needs_proto_lint=true ;; esac + # A protocol-semantic change does not have to alter the protobuf schema. + # Require the capability contract gate for production FSM/admission code + # as well, while allowing focused tests to evolve independently. + case "$path" in + internal/domain/processing/*.go|internal/application/admission/*.go|internal/infra/plan/*.go|internal/infra/membership/*.go|internal/infra/node/*.go) + if [[ "$path" != *_test.go ]]; then + needs_capability_contract=true + fi + ;; + esac + case "$status" in A|D|M) ;; *) needs_full=true ;; @@ -243,6 +259,9 @@ fi if [[ "$needs_proto_lint" == true ]]; then gates+=(proto-lint) fi +if [[ "$needs_capability_contract" == true ]]; then + gates+=(capability-contract) +fi if [[ "$mode" == list ]]; then printf '%s\n' "${gates[@]}" @@ -307,6 +326,9 @@ if [[ "$mode" == normalize ]]; then fi echo "==> agent-check-pr: selected local gates: ${gates[*]}" +if [[ "$needs_capability_contract" == true ]]; then + bash "$SCRIPT_DIR/check-capability-contract" "$base_sha" +fi if [[ "$needs_protocol" == true ]]; then # Cheap and decisive: run before the long suites. PROTOCOL_BASE_SHA="$base_sha" just check-protocol-version diff --git a/scripts/agentcheckpr/selector_test.go b/scripts/agentcheckpr/selector_test.go index d0b1901bb6..8c8c93cbd3 100644 --- a/scripts/agentcheckpr/selector_test.go +++ b/scripts/agentcheckpr/selector_test.go @@ -83,6 +83,16 @@ func TestSelectsLocalValidationGatesFromCompleteDiff(t *testing.T) { paths: []string{"tools/buf-plugin-ledger/main.go"}, expected: []string{"pre-commit", "agent-check", "test-tooling", "proto-lint"}, }, + { + name: "Raft protocol change requires capability contract", + paths: []string{"misc/proto/common.proto"}, + expected: []string{"agent-check-full", "test-e2e", "protocol-revision", "proto-lint", "capability-contract"}, + }, + { + name: "FSM behavior change requires capability contract", + paths: []string{"internal/domain/processing/processor.go"}, + expected: []string{"agent-check-full", "capability-contract"}, + }, { name: "protocol revision only", paths: []string{"pkg/grpcprotocol/protocol.go"}, diff --git a/scripts/check-capability-contract b/scripts/check-capability-contract new file mode 100755 index 0000000000..0c3c653b2e --- /dev/null +++ b/scripts/check-capability-contract @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +base_sha=${1:-${AI_REVIEW_BASE_SHA:-}} +if [[ -z "$base_sha" ]]; then + echo "check-capability-contract: review base SHA is required" >&2 + exit 1 +fi + +mapfile -t changed < <( + git diff --name-only "$base_sha" -- + git ls-files --others --exclude-standard +) +protocol_changed=false +contract_changed=false +for path in "${changed[@]}"; do + case "$path" in + misc/proto/common.proto|misc/proto/raft_cmd.proto) + protocol_changed=true + ;; + internal/domain/processing/*.go|internal/application/admission/*.go|internal/infra/plan/*.go|internal/infra/membership/*.go|internal/infra/node/*.go) + if [[ "$path" != *_test.go ]]; then + protocol_changed=true + fi + ;; + internal/infra/capabilities/*|docs/technical/architecture/subsystems/consensus/capabilities.md) + contract_changed=true + ;; + esac +done + +if [[ "$protocol_changed" == true && "$contract_changed" != true ]]; then + echo "CAPABILITY_CONTRACT=FAIL" >&2 + echo "A Raft/FSM protocol schema changed without a capability registry or consensus capability-document change." >&2 + echo "If the change is backward-compatible, document why in the capability document; otherwise add an ID, derivation, admission/replay tests, and rollout documentation." >&2 + exit 1 +fi + +echo "CAPABILITY_CONTRACT=PASS"