From 5234618e842154ffe0665644938ca32494e96166 Mon Sep 17 00:00:00 2001
From: Saqib <saqibmanan6@gmail.com>
Date: Mon, 17 Aug 2026 21:44:17 +0530
Subject: [PATCH 1/2] feat: gate the ECS deploy on a real smoke-test run

Adds a smoke-tests job calling CivicDataSpace-test's run-smoke.yml,
passing deployed_sha (github.sha) so the gate verifies /health/'s
git_sha field matches, and min_passed to catch a fully-skipped run
looking green. Requires a new repo variable DEV_API_BASE_URL and the
same secrets DataSpaceFrontend's own pipeline already passes to this
workflow (HOME_URL_DEV, TEST_EMAIL_1/2, TEST_PASSWORD_1/2).

Also captures the currently-running task definition ARN before the
service update (deploy-app now has an output for it) -- needed by the
rollback job that follows in the next commit.
---
 .github/workflows/deploy-to-ecs.yml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml
index bab2d4c..9051ae9 100644
--- a/.github/workflows/deploy-to-ecs.yml
+++ b/.github/workflows/deploy-to-ecs.yml
@@ -61,6 +61,8 @@ jobs:
     runs-on: ubuntu-latest
     environment: development
     needs: deploy-infrastructure
+    outputs:
+      previous_task_def_arn: ${{ steps.previous-task-def.outputs.arn }}
 
     steps:
       - name: Checkout
@@ -103,6 +105,17 @@ jobs:
           container-name: dataspace
           image: ${{ steps.build-image.outputs.image }}
 
+      - name: Capture currently-running task definition (for rollback)
+        id: previous-task-def
+        run: |
+          ARN=$(aws ecs describe-services \
+            --cluster "${{ env.ECS_CLUSTER }}" \
+            --services "${{ secrets.ECS_SERVICE }}" \
+            --query 'services[0].taskDefinition' \
+            --output text)
+          echo "Currently running: $ARN"
+          echo "arn=$ARN" >> "$GITHUB_OUTPUT"
+
       - name: Get running service's network configuration
         id: network-config
         run: |
@@ -131,6 +144,21 @@ jobs:
           run-task-assign-public-IP: ${{ steps.network-config.outputs.assign_public_ip }}
           wait-for-task-stopped: true
 
+  smoke-tests:
+    name: Smoke Tests
+    needs: deploy-app
+    uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI
+    with:
+      api_base_url: ${{ vars.DEV_API_BASE_URL }}
+      deployed_sha: ${{ github.sha }}
+      min_passed: 1
+    secrets:
+      HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }}
+      TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }}
+      TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }}
+      TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
+      TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
+
   deploy-otel:
     name: Deploy OpenTelemetry Collector
     runs-on: ubuntu-latest

From ba349a653a2f1aca0a47129e33c56823e84e8217 Mon Sep 17 00:00:00 2001
From: Saqib <saqibmanan6@gmail.com>
Date: Mon, 17 Aug 2026 21:44:45 +0530
Subject: [PATCH 2/2] feat: auto-rollback the ECS service when smoke tests fail

Restores the task definition ARN captured before this deploy's service
update (previous commit), waits for the rolled-back service to
stabilize, then exits 1 -- the run stays red even after successful
mitigation, matching the policy that a rollback is damage control, not
a pass. Migrations applied by the bad deploy are never auto-reverted;
the error message points at where to find what ran.

OTel collector rollback is explicitly out of scope here -- separate
service, independent risk, keeps this change's blast radius to the
app service only.
---
 .github/workflows/deploy-to-ecs.yml | 38 +++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml
index 9051ae9..b18dcc1 100644
--- a/.github/workflows/deploy-to-ecs.yml
+++ b/.github/workflows/deploy-to-ecs.yml
@@ -159,6 +159,44 @@ jobs:
       TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
       TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
 
+  rollback-on-smoke-failure:
+    name: Rollback on Smoke Failure
+    runs-on: ubuntu-latest
+    environment: development
+    needs: [deploy-app, smoke-tests]
+    if: failure() && needs.deploy-app.result == 'success'
+    timeout-minutes: 15
+
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Restore the previously-running task definition
+        run: |
+          PREVIOUS_ARN="${{ needs.deploy-app.outputs.previous_task_def_arn }}"
+          if [ -z "$PREVIOUS_ARN" ]; then
+            echo "::error::No previous task definition was captured -- nothing to roll back to. This is expected on a service's very first deploy."
+            exit 1
+          fi
+          echo "Rolling back to: $PREVIOUS_ARN"
+          aws ecs update-service \
+            --cluster "${{ env.ECS_CLUSTER }}" \
+            --service "${{ secrets.ECS_SERVICE }}" \
+            --task-definition "$PREVIOUS_ARN" \
+            --force-new-deployment
+          aws ecs wait services-stable \
+            --cluster "${{ env.ECS_CLUSTER }}" \
+            --services "${{ secrets.ECS_SERVICE }}"
+
+      - name: Mark this run as failed despite successful rollback
+        run: |
+          echo "::error::Smoke tests failed after deploy. Rolled back to the previous task definition -- migrations applied by this deploy were NOT reverted. Check what ran via the 'Run migrations, then deploy' step's logs before re-deploying."
+          exit 1
+
   deploy-otel:
     name: Deploy OpenTelemetry Collector
     runs-on: ubuntu-latest