From 2ada230e01ca42244e8e386b060614964ec63e39 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:32:58 +0530 Subject: [PATCH 1/6] fix: always run the CloudFormation deploy, don't guess from head_commit github.event.head_commit.modified is only populated for single-commit pushes -- a squash-merge touching aws/cloudformation would silently skip infra sync. The CFN deploy is already idempotent (--no-fail-on-empty-changeset), so simplest robust fix is to just run it every time; costs one extra ~10-20s idempotent call per deploy. --- .github/workflows/deploy-to-ecs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 4fe14158..73a5dbb1 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -27,7 +27,6 @@ jobs: name: Deploy Infrastructure runs-on: ubuntu-latest environment: development - if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.modified, 'aws/cloudformation') steps: - name: Checkout From abeb9d16404a01ea7944918ce3f362b4eeb9f3eb Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:33:29 +0530 Subject: [PATCH 2/6] fix: stop deploy-app running on a cancelled workflow if: always() made this job run even if the workflow was cancelled before it started. Now that deploy-infrastructure always runs (prior commit), it can never legitimately be 'skipped' either, so the implicit needs: gating (success-only, false on cancellation) is exactly the behavior wanted -- no explicit if: needed. --- .github/workflows/deploy-to-ecs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 73a5dbb1..5487b8f6 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -61,7 +61,6 @@ jobs: runs-on: ubuntu-latest environment: development needs: deploy-infrastructure - if: always() # Run even if infrastructure deployment is skipped steps: - name: Checkout From 4761b415a2d06b559a799910f73ffbcc6f826f3a Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:37:45 +0530 Subject: [PATCH 3/6] feat: run migrations as an explicit ECS one-off task before deploy Bumps aws-actions/amazon-ecs-deploy-task-definition v1 -> v2 (purely additive per its changelog) to use its built-in run-task support: runs a standalone task on the new task definition, waits for it to stop, and fails the whole action on a non-zero container exit -- before the service update ever happens. Network config (subnets, security groups, public-IP assignment) is read from the currently running service via describe-services rather than hardcoded, so the migration task always runs in the same network context as the app. --- .github/workflows/deploy-to-ecs.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 5487b8f6..bab2d4c5 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -103,13 +103,33 @@ jobs: container-name: dataspace image: ${{ steps.build-image.outputs.image }} - - name: Deploy main application ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + - name: Get running service's network configuration + id: network-config + run: | + NETWORK_CONFIG=$(aws ecs describe-services \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" \ + --query 'services[0].networkConfiguration.awsvpcConfiguration' \ + --output json) + { + echo "subnets=$(echo "$NETWORK_CONFIG" | jq -r '.subnets | join(",")')" + echo "security_groups=$(echo "$NETWORK_CONFIG" | jq -r '.securityGroups | join(",")')" + echo "assign_public_ip=$(echo "$NETWORK_CONFIG" | jq -r '.assignPublicIp')" + } >> "$GITHUB_OUTPUT" + + - name: Run migrations, then deploy the ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.task-def-app.outputs.task-definition }} service: ${{ secrets.ECS_SERVICE }} cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true + run-task: true + run-task-container-overrides: '[{"name":"dataspace","command":["python","manage.py","migrate","--noinput"]}]' + run-task-subnets: ${{ steps.network-config.outputs.subnets }} + run-task-security-groups: ${{ steps.network-config.outputs.security_groups }} + run-task-assign-public-IP: ${{ steps.network-config.outputs.assign_public_ip }} + wait-for-task-stopped: true deploy-otel: name: Deploy OpenTelemetry Collector From dd9a68c6245c68df4f8bbf732643057636323dbc Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:37:45 +0530 Subject: [PATCH 4/6] chore: drop the now-dead migrations-directory scaffolding mkdir/chmod/touch on api/migrations existed only to let the runtime makemigrations step (removed earlier) write new migration files. migrate doesn't need to write to that directory, just read committed migrations, so this block has been dead weight since makemigrations was dropped. --- docker-entrypoint.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 96033e99..a54011c6 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -39,13 +39,12 @@ while True: time.sleep(2) END -# Ensure migrations directory exists with proper permissions -echo "Ensuring migrations directory exists..." -mkdir -p /code/api/migrations -chmod -R 777 /code/api/migrations -touch /code/api/migrations/__init__.py - -# Run migrations +# Run migrations. In the ECS pipeline this also runs earlier as an explicit +# one-off task before this service is deployed (see deploy-to-ecs.yml) so a +# broken migration fails the deploy loudly and visibly instead of surfacing +# only once containers are already shipping; this call is then a no-op +# (migrate is idempotent). Kept here unconditionally too, since this same +# entrypoint/image is what `docker compose up` uses for local dev. echo "Running migrations..." python manage.py migrate --noinput From 5234618e842154ffe0665644938ca32494e96166 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:44:17 +0530 Subject: [PATCH 5/6] feat: gate the ECS deploy on a real smoke-test run Adds a smoke-tests job calling CivicDataSpace-test's run-smoke.yml, passing deployed_sha (github.sha) so the gate verifies /health/'s git_sha field matches, and min_passed to catch a fully-skipped run looking green. Requires a new repo variable DEV_API_BASE_URL and the same secrets DataSpaceFrontend's own pipeline already passes to this workflow (HOME_URL_DEV, TEST_EMAIL_1/2, TEST_PASSWORD_1/2). Also captures the currently-running task definition ARN before the service update (deploy-app now has an output for it) -- needed by the rollback job that follows in the next commit. --- .github/workflows/deploy-to-ecs.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index bab2d4c5..9051ae96 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -61,6 +61,8 @@ jobs: runs-on: ubuntu-latest environment: development needs: deploy-infrastructure + outputs: + previous_task_def_arn: ${{ steps.previous-task-def.outputs.arn }} steps: - name: Checkout @@ -103,6 +105,17 @@ jobs: container-name: dataspace image: ${{ steps.build-image.outputs.image }} + - name: Capture currently-running task definition (for rollback) + id: previous-task-def + run: | + ARN=$(aws ecs describe-services \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" \ + --query 'services[0].taskDefinition' \ + --output text) + echo "Currently running: $ARN" + echo "arn=$ARN" >> "$GITHUB_OUTPUT" + - name: Get running service's network configuration id: network-config run: | @@ -131,6 +144,21 @@ jobs: run-task-assign-public-IP: ${{ steps.network-config.outputs.assign_public_ip }} wait-for-task-stopped: true + smoke-tests: + name: Smoke Tests + needs: deploy-app + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + api_base_url: ${{ vars.DEV_API_BASE_URL }} + deployed_sha: ${{ github.sha }} + min_passed: 1 + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + deploy-otel: name: Deploy OpenTelemetry Collector runs-on: ubuntu-latest From ba349a653a2f1aca0a47129e33c56823e84e8217 Mon Sep 17 00:00:00 2001 From: Saqib Date: Mon, 17 Aug 2026 21:44:45 +0530 Subject: [PATCH 6/6] feat: auto-rollback the ECS service when smoke tests fail Restores the task definition ARN captured before this deploy's service update (previous commit), waits for the rolled-back service to stabilize, then exits 1 -- the run stays red even after successful mitigation, matching the policy that a rollback is damage control, not a pass. Migrations applied by the bad deploy are never auto-reverted; the error message points at where to find what ran. OTel collector rollback is explicitly out of scope here -- separate service, independent risk, keeps this change's blast radius to the app service only. --- .github/workflows/deploy-to-ecs.yml | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index 9051ae96..b18dcc1b 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -159,6 +159,44 @@ jobs: TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + rollback-on-smoke-failure: + name: Rollback on Smoke Failure + runs-on: ubuntu-latest + environment: development + needs: [deploy-app, smoke-tests] + if: failure() && needs.deploy-app.result == 'success' + timeout-minutes: 15 + + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Restore the previously-running task definition + run: | + PREVIOUS_ARN="${{ needs.deploy-app.outputs.previous_task_def_arn }}" + if [ -z "$PREVIOUS_ARN" ]; then + echo "::error::No previous task definition was captured -- nothing to roll back to. This is expected on a service's very first deploy." + exit 1 + fi + echo "Rolling back to: $PREVIOUS_ARN" + aws ecs update-service \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --service "${{ secrets.ECS_SERVICE }}" \ + --task-definition "$PREVIOUS_ARN" \ + --force-new-deployment + aws ecs wait services-stable \ + --cluster "${{ env.ECS_CLUSTER }}" \ + --services "${{ secrets.ECS_SERVICE }}" + + - name: Mark this run as failed despite successful rollback + run: | + echo "::error::Smoke tests failed after deploy. Rolled back to the previous task definition -- migrations applied by this deploy were NOT reverted. Check what ran via the 'Run migrations, then deploy' step's logs before re-deploying." + exit 1 + deploy-otel: name: Deploy OpenTelemetry Collector runs-on: ubuntu-latest