diff --git a/tests/templates/kuttl/ca-cert/90-cleanup-executor-pods.yaml b/tests/templates/kuttl/ca-cert/90-cleanup-executor-pods.yaml index 5fc48224..3c9c9e9d 100644 --- a/tests/templates/kuttl/ca-cert/90-cleanup-executor-pods.yaml +++ b/tests/templates/kuttl/ca-cert/90-cleanup-executor-pods.yaml @@ -1,18 +1,8 @@ --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster --all -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh diff --git a/tests/templates/kuttl/commons/cleanup-executor-pods.sh b/tests/templates/kuttl/commons/cleanup-executor-pods.sh new file mode 100644 index 00000000..db364177 --- /dev/null +++ b/tests/templates/kuttl/commons/cleanup-executor-pods.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env sh +# Shared teardown for KubernetesExecutor tests: force-deletes leftover DAG task pods so +# that kuttl can delete the namespace within its timeout. Their Vector sidecar ignores +# SIGTERM (it is not PID 1) and would linger for the full 300s grace period. +# The proper fix is in operator-rs (making Vector PID 1 via exec). + +# With --skip-delete the namespace is kept for debugging, so this must not run. kuttl does +# not expose the flag to test steps, hence looking for it in the argv of the kuttl ancestor. +pid=$(ps -o ppid= -p "$$" 2>/dev/null | tr -d '[:space:]') +while [ "${pid:-0}" -ge 1 ]; do + args=$(ps -o args= -p "$pid" 2>/dev/null) + case "$args" in + *kuttl*) + case "$args" in + *--skip-delete*) + echo "kuttl was started with --skip-delete, keeping the AirflowCluster" + exit 0 + ;; + esac + ;; + esac + pid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d '[:space:]') +done + +# shellcheck disable=SC2154 # NAMESPACE is set by kuttl for test-step commands +kubectl delete airflowcluster --all -n "$NAMESPACE" --wait=false 2>/dev/null || true + +if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n "$NAMESPACE" --timeout=120s 2>/dev/null; then + exit 0 +fi +kubectl delete pods -l app.kubernetes.io/name=airflow -n "$NAMESPACE" --grace-period=0 --force 2>/dev/null || true +kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n "$NAMESPACE" --timeout=300s diff --git a/tests/templates/kuttl/external-access/90-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/external-access/90-cleanup-executor-pods.yaml.j2 index f2ad3209..23b70785 100644 --- a/tests/templates/kuttl/external-access/90-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/external-access/90-cleanup-executor-pods.yaml.j2 @@ -1,20 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/ldap/99-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/ldap/99-cleanup-executor-pods.yaml.j2 index f2ad3209..23b70785 100644 --- a/tests/templates/kuttl/ldap/99-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/ldap/99-cleanup-executor-pods.yaml.j2 @@ -1,20 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/logging/90-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/logging/90-cleanup-executor-pods.yaml.j2 index f2ad3209..23b70785 100644 --- a/tests/templates/kuttl/logging/90-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/logging/90-cleanup-executor-pods.yaml.j2 @@ -1,20 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/mount-dags-configmap/90-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/mount-dags-configmap/90-cleanup-executor-pods.yaml.j2 index f2ad3209..23b70785 100644 --- a/tests/templates/kuttl/mount-dags-configmap/90-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-configmap/90-cleanup-executor-pods.yaml.j2 @@ -1,20 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/mount-dags-gitsync/90-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/90-cleanup-executor-pods.yaml.j2 index f2ad3209..23b70785 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/90-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/90-cleanup-executor-pods.yaml.j2 @@ -1,20 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/oidc/90-cleanup-executor-pods.yaml b/tests/templates/kuttl/oidc/90-cleanup-executor-pods.yaml index 3bc075fd..3c9c9e9d 100644 --- a/tests/templates/kuttl/oidc/90-cleanup-executor-pods.yaml +++ b/tests/templates/kuttl/oidc/90-cleanup-executor-pods.yaml @@ -1,18 +1,8 @@ --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh diff --git a/tests/templates/kuttl/opa/90-cleanup-executor-pods.yaml b/tests/templates/kuttl/opa/90-cleanup-executor-pods.yaml index 3bc075fd..3c9c9e9d 100644 --- a/tests/templates/kuttl/opa/90-cleanup-executor-pods.yaml +++ b/tests/templates/kuttl/opa/90-cleanup-executor-pods.yaml @@ -1,18 +1,8 @@ --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh diff --git a/tests/templates/kuttl/overrides/90-cleanup-executor-pods.yaml b/tests/templates/kuttl/overrides/90-cleanup-executor-pods.yaml index 5fc48224..3c9c9e9d 100644 --- a/tests/templates/kuttl/overrides/90-cleanup-executor-pods.yaml +++ b/tests/templates/kuttl/overrides/90-cleanup-executor-pods.yaml @@ -1,18 +1,8 @@ --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster --all -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh diff --git a/tests/templates/kuttl/remote-logging/90-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/remote-logging/90-cleanup-executor-pods.yaml.j2 index f2ad3209..23b70785 100644 --- a/tests/templates/kuttl/remote-logging/90-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/remote-logging/90-cleanup-executor-pods.yaml.j2 @@ -1,20 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/smoke/90-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/smoke/90-cleanup-executor-pods.yaml.j2 index 72c281b9..23b70785 100644 --- a/tests/templates/kuttl/smoke/90-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/smoke/90-cleanup-executor-pods.yaml.j2 @@ -1,25 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). -# -# FIXME: This step also runs when kuttl is called with --skip-delete, so the -# AirflowCluster is torn down even though the namespace is kept for debugging. -# kuttl does not expose --skip-delete to test steps, so there is no clean way to -# guard against it from here. +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/triggerer/90-cleanup-executor-pods.yaml.j2 b/tests/templates/kuttl/triggerer/90-cleanup-executor-pods.yaml.j2 index f2ad3209..23b70785 100644 --- a/tests/templates/kuttl/triggerer/90-cleanup-executor-pods.yaml.j2 +++ b/tests/templates/kuttl/triggerer/90-cleanup-executor-pods.yaml.j2 @@ -1,20 +1,10 @@ {% if test_scenario['values']['executor'] == 'kubernetes' %} --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh {% endif %} diff --git a/tests/templates/kuttl/versioning/90-cleanup-executor-pods.yaml b/tests/templates/kuttl/versioning/90-cleanup-executor-pods.yaml index 3bc075fd..3c9c9e9d 100644 --- a/tests/templates/kuttl/versioning/90-cleanup-executor-pods.yaml +++ b/tests/templates/kuttl/versioning/90-cleanup-executor-pods.yaml @@ -1,18 +1,8 @@ --- -# Force-delete KubernetesExecutor DAG task pods before kuttl deletes the namespace. -# Their Vector sidecar does not respond to SIGTERM (it is not PID 1), so these pods -# sit in Terminating for the full terminationGracePeriodSeconds (300s), blocking -# namespace deletion past kuttl's timeout. -# The proper fix is in operator-rs (making Vector PID 1 via exec). +# Force-delete leftover KubernetesExecutor DAG task pods so that kuttl can delete the +# namespace within its timeout. See the script for the details. apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 600 commands: - - script: | - kubectl delete airflowcluster airflow -n $NAMESPACE --wait=false 2>/dev/null || true - - script: | - if kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=120s 2>/dev/null; then - exit 0 - fi - kubectl delete pods -l app.kubernetes.io/name=airflow -n $NAMESPACE --grace-period=0 --force 2>/dev/null || true - kubectl wait --for=delete pod -l app.kubernetes.io/name=airflow -n $NAMESPACE --timeout=300s + - script: sh ../../../../templates/kuttl/commons/cleanup-executor-pods.sh