TASK-1175: generation-fenced durable queue leases

.github/workflows/ci.yml

@@ -0,0 +1,46 @@
+name: CI
+
+on:
+  workflow_call:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: queue-ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: animus_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d animus_test"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/animus_test
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+      - uses: Swatinem/rust-cache@v2
+      - name: Formatting
+        run: cargo fmt --all -- --check
+      - name: Clippy
+        run: cargo clippy --all-targets --all-features -- -D warnings
+      - name: Tests
+        run: cargo test --all-targets

.github/workflows/release.yml

@@ -1,5 +1,12 @@
 name: Release Binaries
 
+permissions:
+  contents: read
+
+concurrency:
+  group: queue-release-${{ github.ref }}
+  cancel-in-progress: false
+
 on:
   push:
     tags:
@@ -12,8 +19,12 @@ on:
         type: string
 
 jobs:
+  verify:
+    uses: ./.github/workflows/ci.yml
+
   build:
     name: Build (${{ matrix.target }})
+    needs: verify
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -131,9 +142,37 @@ jobs:
           find dist -type f \( -name '*.tar.gz' -o -name '*.tar.gz.sha256' \) ! -path "${ASSETS_DIR}/*" -exec cp {} "${ASSETS_DIR}/" \;
           ls -la "${ASSETS_DIR}"
 
+      - name: Fail closed if the release already exists
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if gh release view "${GITHUB_REF_NAME}" >/dev/null 2>&1; then
+            echo "::error::release ${GITHUB_REF_NAME} already exists; release tags are write-once" >&2
+            exit 1
+          fi
+
       - name: Publish release
         uses: softprops/action-gh-release@v2
         with:
           files: dist/release-assets/*
           fail_on_unmatched_files: true
           generate_release_notes: true
+
+      - name: Verify published assets against bound checksums
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p verify
+          gh release download "${GITHUB_REF_NAME}" --dir verify
+          test "$(find verify -maxdepth 1 -name '*.tar.gz' | wc -l | tr -d ' ')" = "3"
+          test "$(find verify -maxdepth 1 -name '*.tar.gz.sha256' | wc -l | tr -d ' ')" = "3"
+          for checksum in verify/*.tar.gz.sha256; do
+            archive="${checksum%.sha256}"
+            expected="$(tr -d '[:space:]' < "${checksum}")"
+            actual="$(sha256sum "${archive}" | awk '{print $1}')"
+            test "${actual}" = "${expected}"
+          done

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "animus-queue-postgres"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2021"
 license = "Elastic-2.0"
-description = "Durable Postgres-backed queue plugin for Animus — survives daemon restarts/redeploys and reclaims crashed leases. Wire-compatible with animus-queue-default (the `queue` PluginKind)."
+description = "Durable generation-fenced Postgres queue plugin for Animus."
 repository = "https://github.com/launchapp-dev/animus-queue-postgres"
 homepage = "https://github.com/launchapp-dev/animus-cli"
 default-run = "animus-queue-postgres"
@@ -17,14 +17,12 @@ name = "animus-queue-postgres"
 path = "src/main.rs"
 
 [dependencies]
-# Match animus-queue-default v0.3.3's protocol pin EXACTLY (tag v0.5.10 on
-# launchapp-dev/animus-protocol) so the wire types are byte-identical to the
-# reference queue plugin the v0.6.x daemon already talks to. The queue RPC
-# surface is frozen here regardless of the portal's config/subject/chat plugins
-# pinning v0.1.25 — those are a different protocol crate family.
-animus-plugin-protocol  = { git = "https://github.com/launchapp-dev/animus-protocol", tag = "v0.5.10" }
-animus-queue-protocol   = { git = "https://github.com/launchapp-dev/animus-protocol", tag = "v0.5.10" }
-animus-subject-protocol = { git = "https://github.com/launchapp-dev/animus-protocol", tag = "v0.5.10" }
+# Pin the released execution-fence contract exactly. The v2 queue methods are
+# additive; legacy queue/* calls remain available during daemon migration.
+animus-execution-protocol = { git = "https://github.com/launchapp-dev/animus-protocol", tag = "v0.7.0-rc.14" }
+animus-plugin-protocol    = { git = "https://github.com/launchapp-dev/animus-protocol", tag = "v0.7.0-rc.14" }
+animus-queue-protocol     = { git = "https://github.com/launchapp-dev/animus-protocol", tag = "v0.7.0-rc.14" }
+animus-subject-protocol   = { git = "https://github.com/launchapp-dev/animus-protocol", tag = "v0.7.0-rc.14" }
 
 tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "fs", "io-util", "io-std"] }
 serde = { version = "1", features = ["derive"] }

README.md

@@ -1,3 +1,37 @@
 # animus-queue-postgres
 
-Standalone Animus plugin (Postgres-backed).
+Durable Postgres queue plugin for Animus.
+
+Version 0.2 adds the generation-fenced `queue/v2/*` contract used by the
+five-slot coding fleet:
+
+- idempotent enqueue with a monotonic generation per qualified subject;
+- exact repository/base/head-ref reservations;
+- Pending-only fresh leasing (expired assignments are never silently reused);
+- compare-and-swap lease renewal, expired-lease recovery, completion, and
+  return-to-pending;
+- stable workflow and subject identity across daemon restart/recovery;
+- stale-owner fencing after every lease transfer.
+
+Legacy `queue/*` methods remain available for older daemons. They cannot lease
+or mutate v2 entries, so mixed-version deployment fails closed instead of
+creating a second workflow or node.
+
+## Configuration
+
+- `DATABASE_URL` or `ANIMUS_POSTGRES_URL`: Postgres connection URL.
+- `ANIMUS_QUEUE_LEASE_TTL_SECS`: maximum lease TTL (default 1800 seconds).
+- `ANIMUS_QUEUE_TABLE`: safe table identifier (default `queue_item`).
+
+The plugin applies additive, idempotent schema changes. Its health check is not
+ready until both Postgres and the complete v2 schema are available.
+
+## Verification
+
+```sh
+cargo test --all-targets
+cargo clippy --all-targets --all-features -- -D warnings
+```
+
+Postgres integration tests use `TEST_DATABASE_URL`, falling back to
+`postgres://postgres:postgres@localhost:55432/animus_test`.

src/config.rs

@@ -11,7 +11,8 @@ pub const ENV_POSTGRES_URL: &str = "ANIMUS_POSTGRES_URL";
 
 /// Override the lease time-to-live, in seconds. A leased entry whose
 /// `lease_expires_at` (= lease time + this TTL) has passed is reclaimable by a
-/// later `queue/lease`. Default [`DEFAULT_LEASE_TTL_SECS`].
+/// later legacy `queue/lease`. V2 assignments instead require explicit fenced
+/// recovery. Default [`DEFAULT_LEASE_TTL_SECS`].
 pub const ENV_LEASE_TTL_SECS: &str = "ANIMUS_QUEUE_LEASE_TTL_SECS";
 
 /// Override the table name (default `queue_item`). Useful when more than one
@@ -20,7 +21,7 @@ pub const ENV_TABLE: &str = "ANIMUS_QUEUE_TABLE";
 
 /// Default lease TTL: 30 minutes. Long enough that a healthy in-flight
 /// workflow's lease never expires under it, short enough that a crashed
-/// daemon's work is reclaimed promptly on the next lease.
+/// daemon's work becomes eligible for explicit recovery promptly.
 pub const DEFAULT_LEASE_TTL_SECS: i64 = 1800;
 
 /// Default durable queue table name.
@@ -44,16 +45,8 @@ impl QueueConfig {
         let database_url = std::env::var(ENV_DATABASE_URL)
             .ok()
             .filter(|s| !s.is_empty())
-            .or_else(|| {
-                std::env::var(ENV_POSTGRES_URL)
-                    .ok()
-                    .filter(|s| !s.is_empty())
-            })
-            .ok_or_else(|| {
-                anyhow!(
-                    "no Postgres URL configured: set {ENV_DATABASE_URL} (or {ENV_POSTGRES_URL})"
-                )
-            })?;
+            .or_else(|| std::env::var(ENV_POSTGRES_URL).ok().filter(|s| !s.is_empty()))
+            .ok_or_else(|| anyhow!("no Postgres URL configured: set {ENV_DATABASE_URL} (or {ENV_POSTGRES_URL})"))?;
 
         let lease_ttl_secs = std::env::var(ENV_LEASE_TTL_SECS)
             .ok()
@@ -67,11 +60,7 @@ impl QueueConfig {
             .filter(|s| is_safe_ident(s))
             .unwrap_or_else(|| DEFAULT_TABLE.to_string());
 
-        Ok(Self {
-            database_url,
-            lease_ttl_secs,
-            table,
-        })
+        Ok(Self { database_url, lease_ttl_secs, table })
     }
 
     /// In-process builder for tests / embedders.

src/lib.rs

@@ -1,24 +1,22 @@
 //! `animus-queue-postgres`: a durable, Postgres-backed `queue` plugin for
 //! Animus.
 //!
-//! This is a drop-in replacement for `launchapp-dev/animus-queue-default`. It
-//! speaks the EXACT same `queue/*` JSON-RPC contract (from
-//! `animus-queue-protocol` v0.5.10, the tag queue-default v0.3.3 pins) so the
-//! daemon needs no change to use it. The only difference is the storage layer:
-//! the reference plugin keeps queue state in a file-locked JSON blob under the
-//! project root (ephemeral on a container redeploy), whereas this plugin keeps
-//! it in a shared Postgres table (`queue_item`) so the dispatch queue SURVIVES
-//! daemon restarts and container redeploys.
-//!
-//! ## Durability + lease-expiry reclaim
-//!
-//! The headline feature is **lease-expiry reclaim**. Each `queue/lease`
-//! stamps `lease_owner` + `lease_expires_at` on the claimed rows. A `leased`
-//! row whose `lease_expires_at` has passed is treated as re-leasable: the next
-//! `queue/lease` reclaims it via an atomic `SELECT ... FOR UPDATE SKIP LOCKED`.
-//! So if the daemon crashes or the container redeploys mid-workflow, the
-//! unfinished work is re-dispatched instead of being lost (the file backend
-//! would have lost the whole queue).
+//! It retains the legacy `queue/*` surface while adding the generation-fenced
+//! `queue/v2/*` scheduler contract. Queue state and execution ownership live in
+//! shared Postgres tables so daemon restarts do not erase or duplicate work.
+//!
+//! ## Generation-fenced recovery
+//!
+//! V2 enqueue allocates a monotonic generation for a qualified subject and can
+//! reserve one exact repository/head ref. Fresh leasing only considers Pending
+//! rows. When an Assigned lease expires, the daemon must reconcile the existing
+//! workflow and explicitly recover its lease with a compare-and-swap fence.
+//! Recovery preserves workflow/subject generations and increments only lease
+//! generation, so the previous daemon can no longer mutate the entry.
+//!
+//! Legacy leasing keeps its historical expiry-reclaim behavior, but legacy
+//! methods cannot lease or mutate v2 rows. Mixed-version rollouts therefore
+//! fail closed.
 //!
 //! ## State model
 //!
@@ -51,3 +49,6 @@ pub mod store;
 
 pub use config::QueueConfig;
 pub use store::Store;
+
+/// Maximum generation-fenced leases granted in one scheduling call.
+pub const MAX_GENERATION_FENCED_LEASE_BATCH: usize = 5;