Files
fluxer/.github/workflows/deploy-service.yaml
T

474 lines
20 KiB
YAML

# SPDX-License-Identifier: AGPL-3.0-or-later
name: deploy service
on:
workflow_dispatch:
inputs:
service:
description: "Helm chart name to deploy"
type: choice
required: true
options:
- api
- app-proxy
- admin
- docs
- marketing
- media-proxy
- gateway
- messages
- search
- snowflakes
- users
- unfurl
- uploads
- worker
channel:
description: "Release channel (stable or canary)"
type: choice
required: true
options:
- stable
- canary
image-tag:
description: "Docker image tag to deploy (Fluxer CalVer: YYYY.MDD.MICRO)"
type: string
required: true
build-version:
description: "Fluxer CalVer build version to inject into runtime env vars"
type: string
required: false
default: ""
allow-rollback:
description: "Allow deploying an older image tag than the newest GHCR tag"
type: boolean
required: false
default: false
workflow_call:
inputs:
service:
description: "Helm chart name to deploy"
type: string
required: true
channel:
description: "Release channel (stable or canary)"
type: string
required: true
image-tag:
description: "Docker image tag to deploy (Fluxer CalVer: YYYY.MDD.MICRO)"
type: string
required: true
build-version:
description: "Fluxer CalVer build version to inject into runtime env vars"
type: string
required: false
default: ""
allow-rollback:
description: "Allow deploying an older image tag than the newest GHCR tag"
type: boolean
required: false
default: false
secrets:
KUBE_CONFIG:
required: true
GHCR_USERNAME:
required: false
GHCR_TOKEN:
required: false
env:
GHCR_OWNER: ${{ github.repository_owner }}
GHCR_REGISTRY: ghcr.io/${{ github.repository_owner }}
jobs:
deploy:
name: deploy ${{ inputs.service }}
runs-on: ubuntu-24.04
timeout-minutes: 60
environment: ${{ inputs.channel }}
permissions:
contents: read
packages: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: install helm
uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310
- name: configure kubectl
shell: bash
env:
KUBE_CONFIG_B64: ${{ secrets.KUBE_CONFIG }}
run: |
mkdir -p "$HOME/.kube"
printf '%s' "$KUBE_CONFIG_B64" | base64 -d > "$HOME/.kube/config"
chmod 600 "$HOME/.kube/config"
- name: resolve helm args
id: helm
shell: bash
env:
INPUT_SERVICE: ${{ inputs.service }}
INPUT_CHANNEL: ${{ inputs.channel }}
INPUT_IMAGE_TAG: ${{ inputs['image-tag'] }}
INPUT_BUILD_VERSION: ${{ inputs['build-version'] }}
run: |
SERVICE="$INPUT_SERVICE"
CHANNEL="$INPUT_CHANNEL"
TAG="$INPUT_IMAGE_TAG"
BUILD_VERSION="$INPUT_BUILD_VERSION"
GHCR_REGISTRY="${GHCR_REGISTRY:?GHCR_REGISTRY is required}"
if [[ -z "$BUILD_VERSION" ]]; then
BUILD_VERSION="$TAG"
fi
CALVER_RE='^[1-9][0-9]{3}\.[1-9][0-9]{2,3}\.(0|[1-9][0-9]{0,5})$'
if [[ ! "$TAG" =~ $CALVER_RE ]]; then
echo "::error::image-tag must be a Fluxer CalVer tag (YYYY.MDD.MICRO). Channel tags, latest tags, and suffixed tags are not deployable."
exit 1
fi
if [[ ! "$BUILD_VERSION" =~ $CALVER_RE ]]; then
echo "::error::build-version must be a Fluxer CalVer value (YYYY.MDD.MICRO)."
exit 1
fi
CHART_DIR="./deploy/helm/${SERVICE}"
VALUES_ARGS="-f ${CHART_DIR}/values.yaml"
SETS=""
BUILD_PATHS=""
DEPLOY_IMAGE=""
SYNC_WORKER_RELEASE=""
SYNC_WORKER_CHART_DIR=""
SYNC_WORKER_VALUES_ARGS=""
SYNC_WORKER_SETS=""
case "$SERVICE" in
uploads)
if [[ "$CHANNEL" != "stable" ]]; then
echo "::error::uploads deployments are stable-only (single relay serves both channels)."
exit 1
fi
RELEASE="fluxer-uploads"
DEPLOY_IMAGE="fluxer-media-proxy"
SETS="--set-string app.name=uploads --set-string app.image=fluxer-media-proxy --set-string app.tag=${TAG} --set-string app.config=stable"
SETS="${SETS} --set-string app.build.version=${BUILD_VERSION}"
SETS="${SETS} --set-string app.build.channel=stable"
;;
api|app-proxy|admin|docs|marketing)
BASE_IMAGE="fluxer-${SERVICE}"
if [[ "$SERVICE" == "docs" && "$CHANNEL" != "stable" ]]; then
echo "::error::docs deployments are stable-only."
exit 1
fi
if [[ "$CHANNEL" == "canary" ]]; then
NAME="${SERVICE}-canary"
else
NAME="${SERVICE}"
fi
DEPLOY_IMAGE="${BASE_IMAGE}"
RELEASE="fluxer-${SERVICE}-${CHANNEL}"
VALUES_ARGS="${VALUES_ARGS} -f ${CHART_DIR}/values.${CHANNEL}.prod.yaml"
SETS="--set-string app.name=${NAME} --set-string app.image=${DEPLOY_IMAGE} --set-string app.tag=${TAG}"
SETS="${SETS} --set-string app.build.version=${BUILD_VERSION}"
SETS="${SETS} --set-string app.build.channel=${CHANNEL}"
;;
media-proxy)
if [[ "$CHANNEL" != "canary" ]]; then
echo "::error::Media-proxy deployments are only supported on the canary lane."
exit 1
fi
RELEASE="fluxer-${SERVICE}"
DEPLOY_IMAGE="fluxer-media-proxy"
VALUES_ARGS="${VALUES_ARGS} -f ${CHART_DIR}/values.prod.yaml"
SETS="--set-string mediaProxy.image=fluxer-media-proxy --set-string staticProxy.image=fluxer-media-proxy --set-string mediaProxy.tag=${TAG} --set-string staticProxy.tag=${TAG} --set mediaProxy.replicas=16 --set staticProxy.replicas=4 --set-string mediaProxy.nsfwServiceEndpoint=http://int.flx-nyc-misc1.srv.fluxer.dev:8000"
BUILD_PATHS="mediaProxy staticProxy"
;;
gateway)
if [[ "$CHANNEL" != "stable" ]]; then
echo "::error::gateway deployments are stable-only."
exit 1
fi
RELEASE="fluxer-${SERVICE}"
DEPLOY_IMAGE="fluxer-gateway"
VALUES_ARGS="${VALUES_ARGS} -f ${CHART_DIR}/values.prod.yaml"
SETS="--set-string gateway.image=${DEPLOY_IMAGE} --set-string gateway.tag=${TAG}"
BUILD_PATHS="gateway"
;;
worker)
if [[ "$CHANNEL" != "stable" ]]; then
echo "::error::Worker deployments are only supported on the stable lane."
exit 1
fi
RELEASE="fluxer-${SERVICE}"
DEPLOY_IMAGE="fluxer-api"
VALUES_ARGS="${VALUES_ARGS} -f ${CHART_DIR}/values.prod.yaml"
SETS="--set-string workerRealtime.image=fluxer-api --set-string workerUnfurl.image=fluxer-api --set-string workerLifecycle.image=fluxer-api --set-string workerBatch.image=fluxer-api --set-string workerRealtime.tag=${TAG} --set-string workerUnfurl.tag=${TAG} --set-string workerLifecycle.tag=${TAG} --set-string workerBatch.tag=${TAG}"
BUILD_PATHS="workerRealtime workerUnfurl workerLifecycle workerBatch"
;;
messages|search|snowflakes|users|unfurl)
if [[ "$CHANNEL" != "stable" ]]; then
echo "::error::Shared microservice deployments are stable-only; canary traffic selection is done by the callers."
exit 1
fi
DEPLOY_IMAGE="fluxer-${SERVICE}"
RELEASE="fluxer-${SERVICE}"
VALUES_ARGS="${VALUES_ARGS} -f ${CHART_DIR}/values.prod.yaml"
SETS="--set-string svc.image=${DEPLOY_IMAGE} --set-string svc.tag=${TAG}"
SETS="${SETS} --set-string svc.build.version=${BUILD_VERSION}"
SETS="${SETS} --set-string svc.build.channel=stable"
;;
*)
echo "::error::Unknown service chart: ${SERVICE}"
exit 1
;;
esac
for BUILD_PATH in $BUILD_PATHS; do
SETS="${SETS} --set-string ${BUILD_PATH}.build.version=${BUILD_VERSION}"
SETS="${SETS} --set-string ${BUILD_PATH}.build.channel=${CHANNEL}"
done
SETS="--set-string global.registry=${GHCR_REGISTRY} ${SETS}"
if [[ "$SERVICE" == "api" && "$CHANNEL" == "canary" ]]; then
SYNC_WORKER_RELEASE="fluxer-worker"
SYNC_WORKER_CHART_DIR="./deploy/helm/worker"
SYNC_WORKER_VALUES_ARGS="-f ${SYNC_WORKER_CHART_DIR}/values.yaml -f ${SYNC_WORKER_CHART_DIR}/values.prod.yaml"
SYNC_WORKER_SETS="--set-string workerRealtime.image=fluxer-api --set-string workerUnfurl.image=fluxer-api --set-string workerLifecycle.image=fluxer-api --set-string workerBatch.image=fluxer-api"
SYNC_WORKER_SETS="${SYNC_WORKER_SETS} --set-string workerRealtime.tag=${TAG} --set-string workerUnfurl.tag=${TAG} --set-string workerLifecycle.tag=${TAG} --set-string workerBatch.tag=${TAG}"
for BUILD_PATH in workerRealtime workerUnfurl workerLifecycle workerBatch; do
SYNC_WORKER_SETS="${SYNC_WORKER_SETS} --set-string ${BUILD_PATH}.build.version=${BUILD_VERSION}"
SYNC_WORKER_SETS="${SYNC_WORKER_SETS} --set-string ${BUILD_PATH}.build.channel=${CHANNEL}"
done
SYNC_WORKER_SETS="--set-string global.registry=${GHCR_REGISTRY} ${SYNC_WORKER_SETS}"
fi
{
echo "chart-dir=${CHART_DIR}"
echo "release=${RELEASE}"
echo "values-args=${VALUES_ARGS}"
echo "sets=${SETS}"
echo "deploy-image=${DEPLOY_IMAGE}"
echo "deploy-tag=${TAG}"
echo "sync-worker-release=${SYNC_WORKER_RELEASE}"
echo "sync-worker-chart-dir=${SYNC_WORKER_CHART_DIR}"
echo "sync-worker-values-args=${SYNC_WORKER_VALUES_ARGS}"
echo "sync-worker-sets=${SYNC_WORKER_SETS}"
} >> "$GITHUB_OUTPUT"
- name: helm dependency update
shell: bash
run: |
helm dependency update "${{ steps.helm.outputs.chart-dir }}"
if [[ -n "${{ steps.helm.outputs.sync-worker-chart-dir }}" ]]; then
helm dependency update "${{ steps.helm.outputs.sync-worker-chart-dir }}"
fi
- name: prepare docker config
if: steps.helm.outputs.deploy-image != ''
shell: bash
run: |
echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config" >> "$GITHUB_ENV"
mkdir -p "${RUNNER_TEMP}/docker-config"
- name: configure ghcr auth
if: steps.helm.outputs.deploy-image != ''
shell: bash
env:
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ github.token }}
run: |
auth="$(printf '%s:%s' "$GHCR_USERNAME" "$GHCR_TOKEN" | base64 | tr -d '\n')"
printf '{"auths":{"ghcr.io":{"auth":"%s"}}}\n' "$auth" > "$DOCKER_CONFIG/config.json"
- name: verify deploy image exists
if: steps.helm.outputs.deploy-image != ''
shell: bash
run: |
IMAGE_REF="${GHCR_REGISTRY}/${{ steps.helm.outputs.deploy-image }}:${{ steps.helm.outputs.deploy-tag }}"
echo "Verifying ${IMAGE_REF}"
docker manifest inspect "${IMAGE_REF}" > /dev/null
env:
DOCKER_CLI_EXPERIMENTAL: enabled
- name: verify api deploy uses latest image
if: ${{ steps.helm.outputs.deploy-image == 'fluxer-api' && !inputs['allow-rollback'] }}
shell: bash
env:
GH_TOKEN: ${{ github.token }}
GHCR_OWNER: ${{ env.GHCR_OWNER }}
DEPLOY_TAG: ${{ steps.helm.outputs.deploy-tag }}
run: |
set -euo pipefail
CALVER_RE='^[1-9][0-9]{3}\.[1-9][0-9]{2,3}\.(0|[1-9][0-9]{0,5})$'
OWNER_TYPE="$(
curl -fsS \
-H "Authorization: Bearer ${GH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"${GITHUB_API_URL:-https://api.github.com}/repos/${GITHUB_REPOSITORY}" \
| jq -r '.owner.type'
)"
case "$OWNER_TYPE" in
Organization) PACKAGE_OWNER_PATH="orgs/${GHCR_OWNER}" ;;
User) PACKAGE_OWNER_PATH="users/${GHCR_OWNER}" ;;
*)
echo "::error::Unsupported GitHub owner type for package lookup: ${OWNER_TYPE}"
exit 1
;;
esac
LATEST_TAG="$(
curl -fsS \
-H "Authorization: Bearer ${GH_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"${GITHUB_API_URL:-https://api.github.com}/${PACKAGE_OWNER_PATH}/packages/container/fluxer-api/versions?per_page=100" \
| jq -r --arg re "$CALVER_RE" '
[.[].metadata.container.tags[]? |
select(test($re)) |
{tag: ., parts: (split(".") | map(tonumber))}
] | max_by(.parts) | .tag // empty
'
)"
if [[ -z "$LATEST_TAG" ]]; then
echo "::error::Could not resolve the latest fluxer-api CalVer tag from GHCR."
exit 1
fi
if [[ "$DEPLOY_TAG" != "$LATEST_TAG" ]]; then
echo "::error::Refusing to deploy fluxer-api:${DEPLOY_TAG}; latest GHCR tag is fluxer-api:${LATEST_TAG}. Re-run with allow-rollback=true only for an intentional rollback."
exit 1
fi
- name: approve api image for admission policy
if: ${{ inputs.service == 'api' }}
shell: bash
env:
INPUT_CHANNEL: ${{ inputs.channel }}
run: |
DEPLOYMENT="api"
if [[ "$INPUT_CHANNEL" == "canary" ]]; then
DEPLOYMENT="api-canary"
fi
IMAGE_REF="${GHCR_REGISTRY}/${{ steps.helm.outputs.deploy-image }}:${{ steps.helm.outputs.deploy-tag }}"
PREVIOUS_IMAGE="$(kubectl -n fluxer get deployment "$DEPLOYMENT" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true)"
PREVIOUS_TAG=""
if [[ -n "$PREVIOUS_IMAGE" && "$PREVIOUS_IMAGE" != "$IMAGE_REF" && "$PREVIOUS_IMAGE" == *:* ]]; then
PREVIOUS_TAG="${PREVIOUS_IMAGE##*:}"
else
PREVIOUS_IMAGE=""
fi
kubectl -n fluxer create configmap fluxer-api-approved-image \
--from-literal=tag="${{ steps.helm.outputs.deploy-tag }}" \
--from-literal=image="${IMAGE_REF}" \
--from-literal=previousTag="${PREVIOUS_TAG}" \
--from-literal=previousImage="${PREVIOUS_IMAGE}" \
--dry-run=client -o yaml \
| kubectl apply -f -
- name: ensure api admission policy
if: ${{ inputs.service == 'api' }}
shell: bash
run: kubectl apply -f deploy/k8s/fluxer-api-approved-image-policy.yaml
- name: helm upgrade
shell: bash
run: |
RELEASE="${{ steps.helm.outputs.release }}"
CHART_DIR="${{ steps.helm.outputs.chart-dir }}"
VALUES_ARGS="${{ steps.helm.outputs.values-args }}"
SETS="${{ steps.helm.outputs.sets }}"
wait_for_release_idle() {
local release="$1"
local max_checks="$2"
local check=0
local status="unknown"
while (( check < max_checks )); do
check=$((check + 1))
status=$(helm status "$release" -n fluxer -o json 2>/dev/null | jq -r '.info.status // "unknown"' || echo "unknown")
if [[ "$status" != pending-* ]]; then
echo "Release ${release} is ${status}; continuing."
return 0
fi
echo "Release ${release} is ${status}; waiting 10s (${check}/${max_checks})."
sleep 10
done
echo "::warning::Release ${release} still ${status} after ${max_checks} checks; forcing rollback."
if helm rollback "$release" -n fluxer --wait --timeout 5m 2>&1; then
echo "Rollback succeeded; continuing."
return 0
fi
echo "::error::Release ${release} is stuck in ${status} and rollback failed."
return 1
}
helm_upgrade_with_retries() {
local release="$1"
local chart_dir="$2"
local values_args="$3"
local sets="$4"
local values_args_array=()
local sets_array=()
read -r -a values_args_array <<< "$values_args"
read -r -a sets_array <<< "$sets"
wait_for_release_idle "$release" 18
local max_attempts=4
for attempt in $(seq 1 "$max_attempts"); do
echo "Running helm upgrade for ${release}, attempt ${attempt}/${max_attempts}."
set +e
upgrade_output=$(helm upgrade --install "$release" \
"$chart_dir" \
"${values_args_array[@]}" \
-n fluxer \
"${sets_array[@]}" \
--wait --timeout 20m --atomic --history-max 10 2>&1)
exit_code=$?
set -e
printf '%s\n' "$upgrade_output"
if [[ $exit_code -eq 0 ]]; then
return 0
fi
if ! grep -q "another operation (install/upgrade/rollback) is in progress" <<< "$upgrade_output"; then
return "$exit_code"
fi
if [[ $attempt -eq $max_attempts ]]; then
echo "::error::Helm upgrade failed for ${release} after ${max_attempts} attempts because another operation remained in progress."
return "$exit_code"
fi
wait_for_release_idle "$release" 18
done
}
helm_upgrade_with_retries "$RELEASE" "$CHART_DIR" "$VALUES_ARGS" "$SETS"
if [[ -n "${{ steps.helm.outputs.sync-worker-release }}" ]]; then
helm_upgrade_with_retries \
"${{ steps.helm.outputs.sync-worker-release }}" \
"${{ steps.helm.outputs.sync-worker-chart-dir }}" \
"${{ steps.helm.outputs.sync-worker-values-args }}" \
"${{ steps.helm.outputs.sync-worker-sets }}"
fi
- name: seal api admission approved image
if: ${{ success() && inputs.service == 'api' }}
shell: bash
run: |
IMAGE_REF="${GHCR_REGISTRY}/${{ steps.helm.outputs.deploy-image }}:${{ steps.helm.outputs.deploy-tag }}"
kubectl -n fluxer create configmap fluxer-api-approved-image \
--from-literal=tag="${{ steps.helm.outputs.deploy-tag }}" \
--from-literal=image="${IMAGE_REF}" \
--from-literal=previousTag="" \
--from-literal=previousImage="" \
--dry-run=client -o yaml \
| kubectl apply -f -
- name: recover stuck release on failure
if: failure() || cancelled()
shell: bash
run: |
RELEASE="${{ steps.helm.outputs.release }}"
for RELEASE in "$RELEASE" "${{ steps.helm.outputs.sync-worker-release }}"; do
if [[ -z "$RELEASE" ]]; then
continue
fi
STATUS=$(helm status "$RELEASE" -n fluxer -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null || echo "unknown")
if [[ "$STATUS" == "pending-upgrade" || "$STATUS" == "pending-install" || "$STATUS" == "pending-rollback" ]]; then
echo "::warning::Release ${RELEASE} stuck in ${STATUS}, rolling back..."
helm rollback "$RELEASE" -n fluxer --wait --timeout 5m || true
fi
done