Skip to content

Remove automated EKS release path #301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
suguanYang merged 1 commit into from
Aug 18, 2026
Merged

Remove automated EKS release path #301

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
251 changes: 5 additions & 246 deletions .github/workflows/build-images.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,12 @@ on:
- all
- api
- worker
deployment_target:
description: Staging deployment target
required: false
type: choice
default: ecs-staging
options:
- ecs-staging
- eks-staging-rollback

env:
ACR_REGISTRY: ${{ secrets.ALIYUN_ACR_REGISTRY }}
ACR_NAMESPACE: ${{ secrets.ALIYUN_ACR_NAMESPACE }}
ECR_REGISTRY: 107424103509.dkr.ecr.us-east-1.amazonaws.com
ECR_REPOSITORY: knowhere
AWS_EKS_PROD_CLUSTER_NAME: ${{ secrets.AWS_EKS_PROD_CLUSTER_NAME }}
AWS_EKS_PROD_REGION: ${{ secrets.AWS_EKS_PROD_REGION }}
AWS_ECS_STAGING_CLUSTER_NAME: knowhere-fargate
AWS_ECS_STAGING_REGION: us-east-1
AWS_ECS_STAGING_EXECUTION_ROLE_ARN: arn:aws:iam::107424103509:role/knowhere-fargate-staging-execution-role
Expand Down Expand Up @@ -87,8 +77,8 @@ jobs:
with:
persist-credentials: false

# Production releases are cut from main. This keeps the new ECS path
# and the retained EKS rollback image on the same reviewed source line.
# Production releases are cut from main so the ECS path uses a reviewed
# source line.
- name: Verify production release source
if: ${{ github.event_name == 'release' }}
env:
Expand Down Expand Up @@ -119,7 +109,7 @@ jobs:

should_push="true"

if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.deployment_target }}" != "ecs-staging" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then
if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then
should_build="false"
else
should_build="true"
Expand Down Expand Up @@ -266,8 +256,6 @@ jobs:
${{ (github.ref == 'refs/heads/staging' ||
github.event_name == 'workflow_dispatch') &&
(github.event_name != 'workflow_dispatch' ||
github.event.inputs.deployment_target == 'ecs-staging') &&
(github.event_name != 'workflow_dispatch' ||
github.event.inputs.service == '' ||
github.event.inputs.service == 'all' ||
github.event.inputs.service == 'api') }}
Expand Down Expand Up @@ -383,102 +371,6 @@ jobs:
"${image_uri}" \
-m alembic upgrade heads

migrate-eks-staging:
name: Run EKS rollback database migration
runs-on: ubuntu-latest
needs: build-and-publish
if: >-
${{ github.event_name == 'workflow_dispatch' &&
github.event.inputs.deployment_target == 'eks-staging-rollback' &&
(github.event.inputs.service == '' ||
github.event.inputs.service == 'all' ||
github.event.inputs.service == 'api') }}
permissions:
contents: read

steps:
- name: Decide migration context
id: context
shell: bash
run: |
set -euo pipefail
short_sha="${GITHUB_SHA::8}"
echo "namespace=knowhere-staging" >> "$GITHUB_OUTPUT"
echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/knowhere-backend:staging-${short_sha}" >> "$GITHUB_OUTPUT"

- name: Validate deployment configuration
shell: bash
run: |
set -euo pipefail
if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || \
[ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ] || \
[ -z "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" ] || \
[ -z "${{ env.AWS_EKS_PROD_REGION }}" ]; then
echo "::error::EKS rollback migration requires AWS deployment credentials and cluster configuration."
exit 1
fi

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_EKS_PROD_REGION }}

- name: Setup kubectl
uses: azure/setup-kubectl@v3

- name: Update kubeconfig
shell: bash
run: |
aws eks update-kubeconfig \
--name "${AWS_EKS_PROD_CLUSTER_NAME}" \
--region "${AWS_EKS_PROD_REGION}"

- name: Run migration job
shell: bash
env:
IMAGE_URI: ${{ steps.context.outputs.image_uri }}
NAMESPACE: ${{ steps.context.outputs.namespace }}
run: |
set -euo pipefail

job_name="knowhere-api-migrate-${GITHUB_RUN_ID}"
migration_manifest="$(kubectl get deployment/knowhere-api --namespace "$NAMESPACE" -o json \
| jq --arg job_name "$job_name" --arg image_uri "$IMAGE_URI" '
if ((.spec.template.spec.containers // []) | length) == 0 then
error("knowhere-api deployment has no containers")
else
{
apiVersion: "batch/v1",
kind: "Job",
metadata: {name: $job_name, namespace: .metadata.namespace},
spec: {
backoffLimit: 0,
ttlSecondsAfterFinished: 300,
template: {
metadata: {labels: {"app": "knowhere-api-migrate"}},
spec: (
.spec.template.spec
| .restartPolicy = "Never"
| .containers[0].image = $image_uri
| .containers[0].command = ["python", "-m", "alembic", "upgrade", "heads"]
| del(.containers[0].args)
)
}
}
}
end
')"

printf '%s\n' "$migration_manifest" | kubectl apply -f -
if ! kubectl wait --for=condition=complete "job/$job_name" --namespace "$NAMESPACE" --timeout=900s; then
kubectl describe job "$job_name" --namespace "$NAMESPACE" || true
kubectl logs "job/$job_name" --namespace "$NAMESPACE" --all-containers=true || true
exit 1
fi
kubectl delete job "$job_name" --namespace "$NAMESPACE" --ignore-not-found

deploy-ecs-staging:
name: Deploy staging services to ECS
runs-on: ubuntu-latest
Expand All @@ -487,8 +379,7 @@ jobs:
${{ always() && needs.build-and-publish.result == 'success' &&
(needs['migrate-ecs-staging'].result == 'success' || needs['migrate-ecs-staging'].result == 'skipped') &&
((github.ref == 'refs/heads/staging' && github.event_name == 'push') ||
(github.event_name == 'workflow_dispatch' &&
github.event.inputs.deployment_target == 'ecs-staging')) }}
github.event_name == 'workflow_dispatch') }}
permissions:
contents: read

Expand Down Expand Up @@ -946,145 +837,13 @@ jobs:
echo "API image: ${{ steps.images.outputs.api_image }}"
echo "Worker image: ${{ steps.images.outputs.worker_image }}"

deploy-eks:
runs-on: ubuntu-latest
needs: [build-and-publish, migrate-eks-staging, migrate-ecs-production]
if: >-
${{ always() && github.event_name != 'pull_request' &&
needs.build-and-publish.result == 'success' &&
(needs['migrate-eks-staging'].result == 'success' || needs['migrate-eks-staging'].result == 'skipped') &&
(needs['migrate-ecs-production'].result == 'success' || needs['migrate-ecs-production'].result == 'skipped') &&
(github.event_name == 'release' ||
(github.event_name == 'workflow_dispatch' &&
github.event.inputs.deployment_target == 'eks-staging-rollback')) }}
permissions:
contents: read

strategy:
matrix:
service: [api, worker]
include:
- service: api
ecr_repo_name: knowhere-backend
- service: worker
ecr_repo_name: knowhere-worker

steps:
- name: Decide deployment context
id: context
shell: bash
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
environment="staging"
elif [ "${{ github.event_name }}" = "release" ]; then
environment="prod"
elif [[ "${{ github.ref }}" == refs/tags/* ]]; then
environment="prod"
elif [ "${{ github.ref }}" = "refs/heads/staging" ]; then
environment="staging"
else
environment="staging"
should_deploy="false"
fi

if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then
should_deploy="false"
elif [ -z "${should_deploy:-}" ]; then
should_deploy="true"
fi

if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || \
[ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ] || \
[ -z "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" ] || \
[ -z "${{ env.AWS_EKS_PROD_REGION }}" ]; then
if [ "$environment" = "prod" ] && [ "$should_deploy" = "true" ]; then
echo "::error::Production deployment credentials are not configured."
exit 1
fi
should_deploy="false"
fi

if [ "$environment" = "prod" ]; then
namespace="knowhere-prod"
else
namespace="knowhere-staging"
fi

short_sha="${GITHUB_SHA::8}"
if [ "${{ github.event_name }}" = "release" ]; then
git_tag="${{ github.event.release.tag_name }}"
image_tag="${git_tag}-${environment}"
elif [[ "${{ github.ref }}" == refs/tags/* ]]; then
git_tag="${GITHUB_REF#refs/tags/}"
image_tag="${git_tag}-${environment}"
else
image_tag="${environment}-${short_sha}"
fi

echo "environment=$environment" >> "$GITHUB_OUTPUT"
echo "namespace=$namespace" >> "$GITHUB_OUTPUT"
echo "should_deploy=$should_deploy" >> "$GITHUB_OUTPUT"
echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/${{ matrix.ecr_repo_name }}:${image_tag}" >> "$GITHUB_OUTPUT"

- name: Configure AWS credentials
if: steps.context.outputs.should_deploy == 'true'
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_EKS_PROD_REGION }}

- name: Update kubeconfig
if: steps.context.outputs.should_deploy == 'true'
shell: bash
run: |
aws eks update-kubeconfig \
--name "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" \
--region "${{ env.AWS_EKS_PROD_REGION }}"

- name: Roll out service on AWS EKS
if: steps.context.outputs.should_deploy == 'true'
shell: bash
run: |
deployment_name="knowhere-${{ matrix.service }}"
container_name="${{ matrix.service }}"
namespace="${{ steps.context.outputs.namespace }}"
image_uri="${{ steps.context.outputs.image_uri }}"

echo "Deploying ${deployment_name} to ${namespace} with ${image_uri}"

kubectl set image "deployment/${deployment_name}" \
"${container_name}=${image_uri}" \
--namespace "${namespace}"

if [ "${{ matrix.service }}" = "worker" ]; then
kubectl patch "deployment/${deployment_name}" \
--namespace "${namespace}" \
--type='strategic' \
-p='{"spec":{"template":{"spec":{"containers":[{"name":"worker","readinessProbe":{"exec":{"command":["python","-c","from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()"]}},"livenessProbe":{"exec":{"command":["python","-c","from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()"]}}}]}}}}'
fi

kubectl rollout status "deployment/${deployment_name}" \
--namespace "${namespace}" \
--timeout=300s

- name: Summarize deployment
if: steps.context.outputs.should_deploy == 'true'
shell: bash
run: |
echo "Service: ${{ matrix.service }}"
echo "Environment: ${{ steps.context.outputs.environment }}"
echo "Namespace: ${{ steps.context.outputs.namespace }}"
echo "Image: ${{ steps.context.outputs.image_uri }}"

release:
name: Attach deployment release assets
runs-on: ubuntu-latest
needs: [deploy-eks, deploy-ecs-production]
needs: deploy-ecs-production
if: >-
${{ github.event_name == 'release' &&
github.event.action == 'published' &&
needs.deploy-eks.result == 'success' &&
needs.deploy-ecs-production.result == 'success' }}
permissions:
contents: write
Expand Down
Loading