diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f69a4b706..e3c8a9aea 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,7 +7,6 @@ ARG USER_UID=1000 ARG USER_GID=1000 ARG NODE_MAJOR=24 ARG ELP_VERSION=2026-02-27 -ARG HELM_VERSION=4.2.0 ARG PNPM_VERSION=10.29.3 ARG WASM_BINDGEN_VERSION=0.2.122 @@ -15,8 +14,8 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ + acl \ bash \ - brotli \ build-essential \ ca-certificates \ clang \ @@ -42,13 +41,13 @@ RUN apt-get update \ libfido2-dev \ libgbm1 \ libgtk-3-0 \ + libimage-exiftool-perl \ libnotify4 \ libnss3 \ libpipewire-0.3-dev \ libpulse-dev \ libsecret-1-0 \ libudev-dev \ - libuv1-dev \ libcurl4-openssl-dev \ libswresample-dev \ libswscale-dev \ @@ -71,14 +70,12 @@ RUN apt-get update \ ninja-build \ openssh-client \ pkg-config \ - protobuf-compiler \ python3 \ python3-pip \ - rsync \ + python3-venv \ sudo \ rpm \ unzip \ - webp \ xz-utils \ xdg-utils \ zstd \ @@ -90,6 +87,7 @@ RUN apt-get update \ bat \ btop \ docker-cli \ + docker-compose \ dnsutils \ fd-find \ gdb \ @@ -122,24 +120,12 @@ RUN apt-get update \ && ln -sf /usr/bin/batcat /usr/local/bin/bat \ && rm -rf /var/lib/apt/lists/* -RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \ +RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && rm -rf /var/lib/apt/lists/* \ && corepack enable -RUN ARCH="$(dpkg --print-architecture)" \ - && case "$ARCH" in \ - amd64) HELM_ARCH="amd64" ;; \ - arm64) HELM_ARCH="arm64" ;; \ - *) echo "Unsupported architecture for Helm: $ARCH" >&2; exit 1 ;; \ - esac \ - && curl -fsSL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${HELM_ARCH}.tar.gz" -o /tmp/helm.tgz \ - && tar -C /tmp -xzf /tmp/helm.tgz "linux-${HELM_ARCH}/helm" \ - && mv "/tmp/linux-${HELM_ARCH}/helm" /usr/local/bin/helm \ - && chmod +x /usr/local/bin/helm \ - && rm -rf /tmp/helm.tgz "/tmp/linux-${HELM_ARCH}" - -RUN python3 -m pip install --break-system-packages --no-cache-dir awscli cqlsh +RUN python3 -m pip install --break-system-packages --no-cache-dir awscli COPY tools/fonts/requirements.txt /tmp/fluxer-fonts-requirements.txt RUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer-fonts-requirements.txt \ @@ -147,18 +133,13 @@ RUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer && pyftsubset --help >/dev/null \ && python3 -c "import fontTools, brotli" -RUN if ! command -v rebar3 >/dev/null 2>&1; then \ - curl -fsSL https://s3.amazonaws.com/rebar3/rebar3 -o /usr/local/bin/rebar3 \ - && chmod +x /usr/local/bin/rebar3; \ - fi - RUN ARCH="$(dpkg --print-architecture)" \ && case "$ARCH" in \ amd64) ELP_ARCH="x86_64" ;; \ arm64) ELP_ARCH="aarch64" ;; \ *) echo "Unsupported architecture for ELP: $ARCH" >&2; exit 1 ;; \ esac \ - && curl -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \ + && curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \ && tar -C /usr/local/bin -xzf /tmp/elp.tgz elp \ && chmod +x /usr/local/bin/elp \ && rm /tmp/elp.tgz @@ -179,16 +160,13 @@ ENV DOCKER_HOST="unix:///var/run/docker.sock" \ ENV CC_wasm32_unknown_unknown="clang" \ AR_wasm32_unknown_unknown="llvm-ar" -RUN curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile default --component clippy,rustfmt \ +RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --component clippy,rustfmt \ && rustup target add wasm32-unknown-unknown \ && cargo install cargo-watch --locked \ - && cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked + && cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked \ + && rm -rf "/home/${USERNAME}/.cargo/registry" "/home/${USERNAME}/.cargo/git" RUN corepack prepare "pnpm@${PNPM_VERSION}" --activate \ && pnpm --version -RUN sudo apt-get update \ - && sudo apt-get install -y --no-install-recommends python3-venv \ - && sudo rm -rf /var/lib/apt/lists/* - WORKDIR /workspaces/fluxer diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 16a810fb4..42b866038 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,20 +5,17 @@ "workspaceFolder": "/workspaces/fluxer", "shutdownAction": "stopCompose", "remoteUser": "vscode", + "hostRequirements": { + "cpus": 4, + "memory": "8gb", + "storage": "32gb" + }, "remoteEnv": { "DOCKER_HOST": "unix:///var/run/docker.sock" }, "runServices": ["workspace", "postgres", "valkey", "nats", "livekit", "meilisearch", "mailpit"], - "forwardPorts": [ - 3000, 8088, 8080, 8771, 8082, 3010, 3020, 8100, 8101, 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111, - 8112, 8113, 8114, 8115, 8116, 8117, 8118, 8119, 8120, 8121, 8122, 8123, 8124, 8125, 3900, 8888, 9333, 9340, 23646, - 4222, 7700, 7880, 7900, 9200, 8000 - ], + "forwardPorts": [3000, 8088, 8080, 8771, 8082, 8773, 3010, 3020, 8333], "portsAttributes": { - "8000": { - "label": "Zensical docs", - "onAutoForward": "openBrowserOnce" - }, "8088": { "label": "Fluxer dev proxy", "onAutoForward": "notify" @@ -29,36 +26,15 @@ "3020": { "label": "Fluxer admin" }, - "8100": { - "label": "Fluxer Rust service health" - }, - "3900": { + "8333": { "label": "SeaweedFS S3" }, - "8888": { - "label": "SeaweedFS filer" - }, - "9333": { - "label": "SeaweedFS master" - }, - "9340": { - "label": "SeaweedFS volume" - }, - "23646": { - "label": "SeaweedFS admin" - }, - "7880": { - "label": "LiveKit" - }, - "7700": { - "label": "Meilisearch" - }, - "9200": { - "label": "Elasticsearch" + "8773": { + "label": "Fluxer app proxy" } }, - "postCreateCommand": "sudo chown -R vscode:vscode /workspaces/fluxer/target && find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -exec sudo chown -R vscode:vscode {} + && sudo chown -R vscode:vscode /home/vscode/.local/share/pnpm && cargo run -p fluxer-dev -- bootstrap", - "postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && cargo run -p fluxer-dev -- post-start && bash /workspaces/fluxer/fluxer_docs/serve.sh --daemon", + "postCreateCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- bootstrap", + "postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- post-start", "customizations": { "vscode": { "settings": { diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index d6ec6eb44..aec056885 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,5 +1,3 @@ -name: fluxer-dev - services: workspace: build: @@ -7,15 +5,29 @@ services: dockerfile: .devcontainer/Dockerfile command: sleep infinity init: true - env_file: - - ../config/env/development.env environment: - FLUXER_SEARCH_ENGINE: meilisearch - FLUXER_SEARCH_URL: http://meilisearch:7700 - FLUXER_SEARCH_API_KEY: fluxer-dev-meilisearch - FLUXER_POSTGRES_HOST: postgres - FLUXER_SELF_HOSTED: "true" DOCKER_HOST: unix:///var/run/docker.sock + npm_config_store_dir: /home/vscode/.local/share/pnpm/store + FLUXER_PUBLIC_PORT: "${FLUXER_DEV_PROXY_PORT:-8088}" + FLUXER_PUBLIC_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" + FLUXER_API_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api" + FLUXER_API_CLIENT_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api" + FLUXER_APP_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" + FLUXER_GATEWAY_ENDPOINT: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/gateway" + FLUXER_MEDIA_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media" + FLUXER_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" + FLUXER_ADMIN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin" + FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media" + FLUXER_S3_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" + FLUXER_LIVEKIT_URL: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/livekit" + FLUXER_LIVEKIT_INTERNAL_URL: "http://livekit:7880" + FLUXER_LIVEKIT_WEBHOOK_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api/webhooks/livekit" + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media" + FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media" + FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" + FLUXER_ADMIN_OAUTH_REDIRECT_URI: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin/oauth2_callback" + FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: "http://localhost,http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" + PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api" volumes: - ..:/workspaces/fluxer:cached - type: volume @@ -229,6 +241,7 @@ services: volume: nocopy: true - pnpm-store:/home/vscode/.local/share/pnpm/store + - docs-venv:/workspaces/fluxer/fluxer_docs/.venv - cargo-registry:/home/vscode/.cargo/registry - cargo-git:/home/vscode/.cargo/git - rust-target:/workspaces/fluxer/target @@ -236,45 +249,31 @@ services: source: ${FLUXER_DOCKER_SOCKET:-/var/run/docker.sock} target: /var/run/docker.sock ports: - - "${FLUXER_DEV_DOCS_PORT:-8000}:8000" - - "${FLUXER_DEV_RSPACK_PORT:-3000}:3000" - - "${FLUXER_DEV_PROXY_PORT:-8088}:8088" - - "${FLUXER_DEV_APP_PROXY_PORT:-8080}:8080" - - "${FLUXER_DEV_API_PORT:-8771}:8771" - - "${FLUXER_DEV_GATEWAY_PORT:-8082}:8082" - - "${FLUXER_DEV_MARKETING_PORT:-3010}:3010" - - "${FLUXER_DEV_ADMIN_PORT:-3020}:3020" - - "${FLUXER_DEV_RUST_SERVICE_PORTS:-8100-8125}:8100-8125" - - "${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333" - - "${FLUXER_DEV_SEAWEEDFS_FILER_PORT:-8888}:8888" - - "${FLUXER_DEV_SEAWEEDFS_MASTER_PORT:-9333}:9333" - - "${FLUXER_DEV_SEAWEEDFS_VOLUME_PORT:-9340}:9340" - - "${FLUXER_DEV_SEAWEEDFS_ADMIN_PORT:-23646}:23646" + - "127.0.0.1:${FLUXER_DEV_RSPACK_PORT:-3000}:3000" + - "127.0.0.1:${FLUXER_DEV_PROXY_PORT:-8088}:8088" + - "127.0.0.1:${FLUXER_DEV_API_PORT:-8080}:8080" + - "127.0.0.1:${FLUXER_DEV_GATEWAY_PORT:-8771}:8771" + - "127.0.0.1:${FLUXER_DEV_MEDIA_PROXY_PORT:-8082}:8082" + - "127.0.0.1:${FLUXER_DEV_APP_PROXY_PORT:-8773}:8773" + - "127.0.0.1:${FLUXER_DEV_MARKETING_PORT:-3010}:3010" + - "127.0.0.1:${FLUXER_DEV_ADMIN_PORT:-3020}:3020" + - "127.0.0.1:${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333" depends_on: - - postgres - - valkey - - nats - - livekit - - meilisearch - - mailpit + postgres: + condition: service_healthy + valkey: + condition: service_started + nats: + condition: service_started + livekit: + condition: service_started + meilisearch: + condition: service_healthy + mailpit: + condition: service_started extra_hosts: - "host.docker.internal:host-gateway" - cassandra: - image: cassandra:5.0.8 - profiles: - - full - environment: - CASSANDRA_CLUSTER_NAME: fluxer-dev - CASSANDRA_DC: datacenter1 - CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch - HEAP_NEWSIZE: 128M - MAX_HEAP_SIZE: 768M - volumes: - - cassandra-data:/var/lib/cassandra - ports: - - "${FLUXER_DEV_CASSANDRA_PORT:-9042}:9042" - postgres: image: postgres:16-alpine environment: @@ -284,13 +283,19 @@ services: volumes: - postgres-data:/var/lib/postgresql/data ports: - - "${FLUXER_DEV_POSTGRES_PORT:-5432}:5432" + - "127.0.0.1:${FLUXER_DEV_POSTGRES_PORT:-5432}:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U fluxer -d fluxer"] + interval: 2s + timeout: 5s + retries: 30 + start_period: 5s valkey: image: valkey/valkey:8.1.7-alpine command: ["valkey-server", "--save", "", "--appendonly", "no"] ports: - - "${FLUXER_DEV_VALKEY_PORT:-6379}:6379" + - "127.0.0.1:${FLUXER_DEV_VALKEY_PORT:-6379}:6379" nats: image: nats:2.14.2-alpine @@ -298,33 +303,22 @@ services: volumes: - nats-data:/data ports: - - "${FLUXER_DEV_NATS_PORT:-4222}:4222" - - "${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222" + - "127.0.0.1:${FLUXER_DEV_NATS_PORT:-4222}:4222" + - "127.0.0.1:${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222" livekit: image: livekit/livekit-server:v1.12.0 command: ["--config", "/etc/livekit.yaml", "--bind", "0.0.0.0"] + environment: + LIVEKIT_RTC_TCP_PORT: "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}" + LIVEKIT_RTC_UDP_PORT_START: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}" + LIVEKIT_RTC_UDP_PORT_END: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}" volumes: - ./livekit.yaml:/etc/livekit.yaml:ro ports: - - "${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880" - - "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:7881" - - "${FLUXER_DEV_LIVEKIT_UDP_PORTS:-7882-7892}:7882-7892/udp" - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:9.3.2 - profiles: - - full - environment: - discovery.type: single-node - xpack.security.enabled: "true" - xpack.security.http.ssl.enabled: "false" - ELASTIC_PASSWORD: fluxer-dev-elasticsearch - ES_JAVA_OPTS: "-Xms512m -Xmx512m" - volumes: - - elasticsearch-data:/usr/share/elasticsearch/data - ports: - - "${FLUXER_DEV_ELASTICSEARCH_PORT:-9200}:9200" + - "127.0.0.1:${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880" + - "127.0.0.1:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}" + - "127.0.0.1:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}/udp" meilisearch: image: getmeili/meilisearch:v1.12 @@ -334,7 +328,13 @@ services: volumes: - meilisearch-data:/meili_data ports: - - "${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700" + - "127.0.0.1:${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700" + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:7700/health"] + interval: 2s + timeout: 5s + retries: 30 + start_period: 5s mailpit: image: axllent/mailpit:v1.30 @@ -349,6 +349,7 @@ services: volumes: pnpm-store: + docs-venv: root-node-modules: fluxer-api-node-modules: fluxer-app-node-modules: @@ -394,9 +395,7 @@ volumes: cargo-registry: cargo-git: rust-target: - cassandra-data: nats-data: - elasticsearch-data: meilisearch-data: mailpit-data: postgres-data: diff --git a/.devcontainer/fix-docker-socket.sh b/.devcontainer/fix-docker-socket.sh index e0c3fdf59..ea864717f 100755 --- a/.devcontainer/fix-docker-socket.sh +++ b/.devcontainer/fix-docker-socket.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # SPDX-License-Identifier: AGPL-3.0-or-later -set -uo pipefail +set -euo pipefail SOCKET="${DOCKER_SOCKET:-/var/run/docker.sock}" -USER_NAME="${USER:-vscode}" +USER_NAME="$(id -un)" if [ ! -S "$SOCKET" ]; then echo "fix-docker-socket: no socket at $SOCKET; skipping (Docker-in-devcontainer will not work)" @@ -16,31 +16,11 @@ if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then exit 0 fi -socket_gid="$(stat -c '%g' "$SOCKET" 2>/dev/null || echo "")" -if [ -z "$socket_gid" ]; then - echo "fix-docker-socket: could not stat $SOCKET; skipping" >&2 - exit 0 -fi - -sudo sh -c ' - set -e - gid="$1" - user="$2" - socket="$3" - if ! getent group "$gid" >/dev/null 2>&1; then - groupadd --gid "$gid" docker-host - fi - group_name="$(getent group "$gid" | cut -d: -f1)" - usermod --append --groups "$group_name" "$user" - chgrp "$gid" "$socket" - chmod g+rw "$socket" -' sh "$socket_gid" "$USER_NAME" "$SOCKET" || { - echo "fix-docker-socket: could not adjust $SOCKET; run docker with sudo" >&2 - exit 0 -} +sudo setfacl --modify "user:${USER_NAME}:rw" "$SOCKET" if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then - echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME (gid $socket_gid)" + echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME" else - echo "fix-docker-socket: $SOCKET still unreachable as $USER_NAME; run docker with sudo" >&2 + echo "fix-docker-socket: $SOCKET is still unreachable as $USER_NAME" >&2 + exit 1 fi diff --git a/.devcontainer/fix-workspace-permissions.sh b/.devcontainer/fix-workspace-permissions.sh new file mode 100644 index 000000000..0ef0c5d4e --- /dev/null +++ b/.devcontainer/fix-workspace-permissions.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -euo pipefail + +owner="$(id -u):$(id -g)" + +repair_tree() { + local path="$1" + local unwritable + if [ ! -d "$path" ]; then + echo "fix-workspace-permissions: expected mount is missing: $path" >&2 + exit 1 + fi + unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)" + if [ ! -w "$path" ] || [ -n "$unwritable" ]; then + sudo find "$path" -xdev \( -type d -o -type f \) -exec chown "$owner" {} + + fi + unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)" + if [ ! -w "$path" ] || [ -n "$unwritable" ]; then + echo "fix-workspace-permissions: $path is not writable as $(id -un)" >&2 + exit 1 + fi +} + +for path in \ + /workspaces/fluxer/target \ + /home/vscode/.cargo/registry \ + /home/vscode/.cargo/git \ + /home/vscode/.local \ + /home/vscode/.local/share/pnpm/store \ + /workspaces/fluxer/fluxer_docs/.venv; do + repair_tree "$path" +done + +while IFS= read -r -d '' path; do + if mountpoint -q "$path"; then + repair_tree "$path" + fi +done < <(find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -print0) diff --git a/.devcontainer/livekit.yaml b/.devcontainer/livekit.yaml index 552df10d2..f49d40c42 100644 --- a/.devcontainer/livekit.yaml +++ b/.devcontainer/livekit.yaml @@ -1,11 +1,10 @@ port: 7880 keys: - devkey: secret + devkey: fluxer-livekit-development-secret rtc: tcp_port: 7881 - udp_port: 7882-7892 node_ip: 127.0.0.1 use_mdns: true stun_servers: diff --git a/config/env/development.env b/config/env/development.env index 5cf6d0547..7a4867130 100644 --- a/config/env/development.env +++ b/config/env/development.env @@ -30,12 +30,6 @@ FLUXER_POSTGRES_PASSWORD=fluxer FLUXER_POSTGRES_SSL=false FLUXER_POSTGRES_MAX_CONNECTIONS=20 FLUXER_POSTGRES_KV_TABLE=fluxer_kv -FLUXER_CASSANDRA_HOSTS=cassandra -FLUXER_CASSANDRA_PORT=9042 -FLUXER_CASSANDRA_KEYSPACE=fluxer -FLUXER_CASSANDRA_LOCAL_DC=datacenter1 -FLUXER_CASSANDRA_USERNAME=fluxer -FLUXER_CASSANDRA_PASSWORD=fluxer FLUXER_KV_URL=redis://valkey:6379/0 FLUXER_NATS_URL=nats://nats:4222 FLUXER_NATS_JETSTREAM_URL=nats://nats:4222 @@ -66,8 +60,9 @@ FLUXER_S3_BUCKET_STATIC=fluxer-static FLUXER_LIVEKIT_ENABLED=true FLUXER_LIVEKIT_URL=ws://localhost:8088/livekit +FLUXER_LIVEKIT_INTERNAL_URL=http://localhost:7880 FLUXER_LIVEKIT_API_KEY=devkey -FLUXER_LIVEKIT_API_SECRET=secret +FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secret FLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekit FLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686} diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml index d3484fe61..57fc260ad 100644 --- a/deploy/self-hosting/docker-compose.yml +++ b/deploy/self-hosting/docker-compose.yml @@ -47,6 +47,7 @@ x-fluxer-env: &fluxer-env FLUXER_LIVEKIT_ENABLED: "true" FLUXER_LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env} FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env} + FLUXER_LIVEKIT_INTERNAL_URL: http://livekit:7880 FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit FLUXER_LIVEKIT_DEFAULT_REGION: '{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}' diff --git a/fluxer_api/src/api/App.ts b/fluxer_api/src/api/App.ts index a35882ba5..9d982b505 100644 --- a/fluxer_api/src/api/App.ts +++ b/fluxer_api/src/api/App.ts @@ -39,7 +39,7 @@ function AbuseAwareAppErrorHandler(err: Error, ctx: Context): Response export async function createAPIApp(options: CreateAPIAppOptions): Promise { const {config, logger} = options; - const shutdownApiLifecycle = createShutdown(logger); + const shutdownApiLifecycle = createShutdown(config, logger); setIsDevelopment(config.nodeEnv === 'development'); const routes = new Hono({strict: true}); configureMiddleware(routes, { diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts index e32b81674..422a4158e 100644 --- a/fluxer_api/src/api/Config.ts +++ b/fluxer_api/src/api/Config.ts @@ -306,6 +306,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { apiSecret: master.integrations.voice.api_secret, webhookUrl: master.integrations.voice.webhook_url, url: master.integrations.voice.url, + internalUrl: master.integrations.voice.internal_url, defaultRegion: master.integrations.voice.default_region, }, stripe: { diff --git a/fluxer_api/src/api/app/APILifecycle.ts b/fluxer_api/src/api/app/APILifecycle.ts index 2cfd66fee..9097416d0 100644 --- a/fluxer_api/src/api/app/APILifecycle.ts +++ b/fluxer_api/src/api/app/APILifecycle.ts @@ -36,6 +36,11 @@ import {JetStreamWorkerQueue} from '../worker/JetStreamWorkerQueue'; import {WorkerService} from '../worker/WorkerService'; let jsConnectionManager: JetStreamConnectionManager | null = null; + +function unsupportedDatabaseBackend(backend: never): never { + throw new Error(`Unsupported database backend during shutdown: ${String(backend)}`); +} + export function createInitializer(config: APIConfig, logger: ILogger): () => Promise { return async (): Promise => { try { @@ -131,7 +136,7 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro try { const kvDeletionQueue = getKVAccountDeletionQueue(); if (await kvDeletionQueue.needsRebuild()) { - logger.warn('KV deletion queue needs rebuild, rebuilding...'); + logger.info('KV deletion queue needs rebuild, rebuilding...'); await kvDeletionQueue.rebuildState(); } else { logger.info('KV deletion queue state is healthy'); @@ -204,13 +209,13 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro logger.info('API service initialization complete'); } catch (error) { logger.error({error}, 'API service initialization failed'); - await createShutdown(logger)(); + await createShutdown(config, logger)(); throw error; } }; } -export function createShutdown(logger: ILogger): () => Promise { +export function createShutdown(config: APIConfig, logger: ILogger): () => Promise { return async (): Promise => { logger.info('Shutting down API service...'); if (jsConnectionManager) { @@ -258,18 +263,26 @@ export function createShutdown(logger: ILogger): () => Promise { } catch (error) { logger.error({error}, 'Error shutting down report service'); } - try { - setDatabaseQueryExecutor(null); - await shutdownPostgres(); - logger.info('Postgres client shut down'); - } catch (error) { - logger.error({error}, 'Error shutting down Postgres client'); - } - try { - await shutdownCassandra(); - logger.info('Cassandra client shut down'); - } catch (error) { - logger.error({error}, 'Error shutting down Cassandra client'); + setDatabaseQueryExecutor(null); + switch (config.database.backend) { + case 'postgres': + try { + await shutdownPostgres(); + logger.info('Postgres client shut down'); + } catch (error) { + logger.error({error}, 'Error shutting down Postgres client'); + } + break; + case 'cassandra': + try { + await shutdownCassandra(); + logger.info('Cassandra client shut down'); + } catch (error) { + logger.error({error}, 'Error shutting down Cassandra client'); + } + break; + default: + unsupportedDatabaseBackend(config.database.backend); } logger.info('API service shutdown complete'); }; diff --git a/fluxer_api/src/api/app/ControllerRegistry.ts b/fluxer_api/src/api/app/ControllerRegistry.ts index 4db31caa0..28c851d9b 100644 --- a/fluxer_api/src/api/app/ControllerRegistry.ts +++ b/fluxer_api/src/api/app/ControllerRegistry.ts @@ -66,7 +66,9 @@ export function registerControllers(routes: HonoApp, config: APIConfig): void { TestHarnessController(routes); } UserController(routes); - registerInboundSmsWebhook(routes); + if (config.sms.enabled) { + registerInboundSmsWebhook(routes); + } WebhookController(routes); OAuth2Controller(routes); OAuth2ApplicationsController(routes); diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts index 3b685ce92..58ea5fd50 100644 --- a/fluxer_api/src/api/config/APIConfig.ts +++ b/fluxer_api/src/api/config/APIConfig.ts @@ -204,6 +204,7 @@ export interface APIConfig { apiSecret?: string; webhookUrl?: string; url?: string; + internalUrl?: string; defaultRegion?: { id: string; name: string; diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts index db05aea5e..8ae05ca42 100644 --- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts +++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts @@ -19,6 +19,8 @@ interface PageState { const VALUE_SEPARATOR = '\u001f'; const ENCODED_TYPE_KEY = '__fluxer_type'; +const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56; +const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s'; function normalizeCql(cql: string): string { return cql.replace(/\s+/g, ' ').trim(); @@ -354,8 +356,13 @@ function parseEqWhere(whereSql: string, cql: string): ReadonlyArray } export async function ensurePostgresKvSchema(client: IPostgresClient): Promise { - const table = quoteIdentifier(client.kvTable()); - await client.query(` + const kvTable = client.kvTable(); + const table = quoteIdentifier(kvTable); + await client.transaction(async (db) => { + await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_LOCK_TIMEOUT]); + await db.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, kvTable]); + await db.query("SELECT set_config('statement_timeout', '0', true)"); + await db.query(` CREATE TABLE IF NOT EXISTS ${table} ( table_name text NOT NULL, partition_key text NOT NULL, @@ -365,28 +372,29 @@ CREATE TABLE IF NOT EXISTS ${table} ( updated_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (table_name, row_key) )`); - await client.query( - `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`, - ); - await client.query( - `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`, - ); - await client.query( - `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`, - ); - await client.query( - `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`, - ); - await client.query( - `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`, - ); - await client.query(` + await db.query( + `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`, + ); + await db.query( + `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`, + ); + await db.query( + `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`, + ); + await db.query( + `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`, + ); + await db.query( + `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`, + ); + await db.query(` UPDATE ${table} SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2) WHERE table_name = 'messages' AND partition_key = row_key AND split_part(row_key, chr(31), 3) <> ''`); - await client.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_idx`)}`); + await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`); + }); } export async function pruneExpiredPostgresKvRows(client: IPostgresClient, batchSize = 5000): Promise { diff --git a/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts b/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts index ec0174ad9..661175056 100644 --- a/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts +++ b/fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts @@ -60,7 +60,7 @@ export class KVAccountDeletionQueueService { } async rebuildState(): Promise { - Logger.info('Starting deletion queue rebuild from Cassandra'); + Logger.info('Starting deletion queue rebuild from primary database'); try { await this.kvClient.del(QUEUE_KEY); await this.kvClient.del(STATE_VERSION_KEY); diff --git a/fluxer_api/src/api/infrastructure/LiveKitService.ts b/fluxer_api/src/api/infrastructure/LiveKitService.ts index 8723c1443..597915364 100644 --- a/fluxer_api/src/api/infrastructure/LiveKitService.ts +++ b/fluxer_api/src/api/infrastructure/LiveKitService.ts @@ -109,6 +109,18 @@ function createRoomServiceClient(endpoint: string, apiKey: string, apiSecret: st return client; } +function resolveRoomServiceEndpoint(server: VoiceServerRecord): string { + const defaultRegion = Config.voice.defaultRegion; + const internalUrl = Config.voice.internalUrl; + if (!defaultRegion || !internalUrl) { + return server.endpoint; + } + if (server.regionId !== defaultRegion.id || server.serverId !== `${defaultRegion.id}-server-1`) { + return server.endpoint; + } + return internalUrl; +} + export class LiveKitService extends ILiveKitService { private serverClients: Map> = new Map(); private topology: VoiceTopology; @@ -453,12 +465,13 @@ export class LiveKitService extends ILiveKitService { const servers = this.topology.getServersForRegion(region.id); const serverMap: Map = new Map(); for (const server of servers) { + const roomServiceEndpoint = resolveRoomServiceEndpoint(server); serverMap.set(server.serverId, { endpoint: server.endpoint, apiKey: server.apiKey, apiSecret: server.apiSecret, isActive: server.isActive, - roomServiceClient: createRoomServiceClient(server.endpoint, server.apiKey, server.apiSecret), + roomServiceClient: createRoomServiceClient(roomServiceEndpoint, server.apiKey, server.apiSecret), }); } newMap.set(region.id, serverMap); diff --git a/fluxer_api/src/api/middleware/ServiceMiddleware.ts b/fluxer_api/src/api/middleware/ServiceMiddleware.ts index 5fb8945aa..ca44b7db2 100644 --- a/fluxer_api/src/api/middleware/ServiceMiddleware.ts +++ b/fluxer_api/src/api/middleware/ServiceMiddleware.ts @@ -284,7 +284,7 @@ export function setInjectedAccountPolicyEvaluator(evaluator: IAccountPolicyEvalu function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator { if (_registrationRiskEvaluator) return _registrationRiskEvaluator; if (!Config.risk.enabled) { - Logger.warn( + Logger.info( {}, '[ServiceMiddleware] integrations.risk_integration.enabled is false — account risk scoring is disabled', ); diff --git a/fluxer_app_proxy/src/routes/assets_proxy.rs b/fluxer_app_proxy/src/routes/assets_proxy.rs index 7d516d388..c2ed95abc 100644 --- a/fluxer_app_proxy/src/routes/assets_proxy.rs +++ b/fluxer_app_proxy/src/routes/assets_proxy.rs @@ -148,7 +148,7 @@ pub async fn proxy_assets( response } -async fn serve_local_asset( +pub(super) async fn serve_local_asset( static_dir: &str, relative_path: &str, request_headers: &HeaderMap, diff --git a/fluxer_app_proxy/src/routes/spa_index.rs b/fluxer_app_proxy/src/routes/spa_index.rs index 487765422..96cfce35d 100644 --- a/fluxer_app_proxy/src/routes/spa_index.rs +++ b/fluxer_app_proxy/src/routes/spa_index.rs @@ -19,6 +19,7 @@ use axum::{ use std::path::Path; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use super::assets_proxy::serve_local_asset; use super::spa_static::{CORS_ALLOW_ANY_VALUE, guess_mime, is_font_mime}; const ACCEPT_CH_VALUE: &str = "DPR, Sec-CH-DPR, Sec-CH-Width, Save-Data, ECT, Downlink"; @@ -35,6 +36,14 @@ pub async fn spa_catch_all( if let Some(cache_control) = static_root_file_cache_control(request_path) { return serve_static_file(&state.config.static_dir, request_path, cache_control).await; } + if is_static_asset_path(request_path) { + return serve_local_asset( + &state.config.static_dir, + request_path.trim_start_matches('/'), + &headers, + ) + .await; + } serve_spa_index(&state, &headers, request_path).await } @@ -42,6 +51,16 @@ pub async fn spa_catch_all( const CRAWL_CONTROL_CACHE_CONTROL: &str = "public, max-age=300, must-revalidate"; const STATIC_ROOT_FILES: &[(&str, &str)] = &[("/robots.txt", CRAWL_CONTROL_CACHE_CONTROL)]; +const STATIC_ASSET_PREFIXES: &[&str] = &[ + "/avatars/", + "/badges/", + "/desktop/", + "/embeds/", + "/emoji/", + "/libs/", + "/marketing/", + "/web/", +]; fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> { STATIC_ROOT_FILES @@ -50,6 +69,12 @@ fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> { .map(|(_, cache_control)| *cache_control) } +fn is_static_asset_path(request_path: &str) -> bool { + STATIC_ASSET_PREFIXES + .iter() + .any(|prefix| request_path.starts_with(prefix)) +} + async fn serve_static_file( static_dir: &str, request_path: &str, diff --git a/fluxer_docs/serve.sh b/fluxer_docs/serve.sh index 8cb5c270a..d744c9067 100755 --- a/fluxer_docs/serve.sh +++ b/fluxer_docs/serve.sh @@ -3,6 +3,7 @@ set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" VENV="$HERE/.venv" +REQUIREMENTS_STAMP="$VENV/.requirements.sha256" ADDR="${ZENSICAL_DEV_ADDR:-0.0.0.0:8000}" LOG="${ZENSICAL_LOG:-/tmp/zensical-serve.log}" PORT="${ADDR##*:}" @@ -14,29 +15,34 @@ ensure_env() { python3 -m venv "$VENV" "$VENV/bin/python" -m pip install --quiet --upgrade pip fi - "$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt" + python_fingerprint="$(python3 -c 'import platform, sys; print(f"{sys.implementation.cache_tag}:{platform.machine()}")')" + requirements_hash="$(printf '%s\0%s\n' "$python_fingerprint" "$(sha256sum "$HERE/requirements.txt" | cut -d' ' -f1)" | sha256sum | cut -d' ' -f1)" + if [ ! -f "$REQUIREMENTS_STAMP" ] || [ "$(cat "$REQUIREMENTS_STAMP")" != "$requirements_hash" ]; then + "$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt" + printf '%s\n' "$requirements_hash" >"$REQUIREMENTS_STAMP" + fi } case "${1:-serve}" in - --bootstrap) - ensure_env - ;; - --daemon) - ensure_env - if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then - echo "zensical already serving on ${ADDR}" - exit 0 - fi - setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 & - disown 2>/dev/null || true - echo "zensical serving on ${ADDR} (logs: ${LOG})" - ;; - serve) - ensure_env - exec "$VENV/bin/zensical" serve -a "$ADDR" - ;; - *) - ensure_env - exec "$VENV/bin/zensical" "$@" - ;; +--bootstrap) + ensure_env + ;; +--daemon) + ensure_env + if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then + echo "zensical already serving on ${ADDR}" + exit 0 + fi + setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 & + disown 2>/dev/null || true + echo "zensical serving on ${ADDR} (logs: ${LOG})" + ;; +serve) + ensure_env + exec "$VENV/bin/zensical" serve -a "$ADDR" + ;; +*) + ensure_env + exec "$VENV/bin/zensical" "$@" + ;; esac diff --git a/fluxer_gateway/Dockerfile b/fluxer_gateway/Dockerfile index 0e715a505..32ff1da63 100644 --- a/fluxer_gateway/Dockerfile +++ b/fluxer_gateway/Dockerfile @@ -12,22 +12,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ curl \ make \ - cmake \ gcc \ g++ \ libc6-dev \ - libssl-dev \ - libuv1-dev \ - zlib1g-dev \ - pkg-config \ gettext-base \ ca-certificates \ && rm -rf /var/lib/apt/lists/* -RUN curl -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \ +RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \ chmod +x /usr/local/bin/rebar3 -RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \ +RUN curl --retry 5 --retry-delay 2 --retry-all-errors --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \ sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}" COPY . . @@ -47,8 +42,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ libstdc++6 \ - libuv1t64 \ - zlib1g \ && rm -rf /var/lib/apt/lists/* COPY --from=build /usr/src/app/fluxer_gateway/_build/prod/rel/fluxer_gateway . diff --git a/fluxer_gateway/rebar.config b/fluxer_gateway/rebar.config index 99a565e7b..de42094df 100644 --- a/fluxer_gateway/rebar.config +++ b/fluxer_gateway/rebar.config @@ -6,7 +6,6 @@ {jose, "1.11.10"}, {ezstd, "1.1.0"}, {enats, "1.2.0"}, - {erlcass, "4.1.3"}, {eqwalizer_support, {git_subdir, "https://github.com/whatsapp/eqwalizer.git", {ref, "c57aa9e3a05553d1ba66ea647a2177eaa469211b"}, "eqwalizer_support"}} @@ -32,7 +31,6 @@ {relx, [ {release, {fluxer_gateway, "0.0.0"}, [ fluxer_gateway, - {erlcass, load}, enats, sasl ]}, @@ -90,7 +88,7 @@ ]}, {plt_apps, all_deps}, {plt_extra_apps, [ - crypto, enats, erlcass, inets, jose, public_key, ranch, ssl + crypto, enats, inets, jose, public_key, ranch, ssl ]}, {warnings_file, "dialyzer.ignore-warnings"} ]}. diff --git a/fluxer_gateway/rebar.lock b/fluxer_gateway/rebar.lock index 2e2d2b7bf..e193fe0ff 100644 --- a/fluxer_gateway/rebar.lock +++ b/fluxer_gateway/rebar.lock @@ -9,7 +9,6 @@ {ref,"c57aa9e3a05553d1ba66ea647a2177eaa469211b"}, "eqwalizer_support"}, 0}, - {<<"erlcass">>,{pkg,<<"erlcass">>,<<"4.1.3">>},0}, {<<"ezstd">>,{pkg,<<"ezstd">>,<<"1.1.0">>},0}, {<<"jose">>,{pkg,<<"jose">>,<<"1.11.10">>},0}, {<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.4.0">>},1}, @@ -21,7 +20,6 @@ {<<"cowlib">>, <<"54592074EBBBB92EE4746C8A8846E5605052F29309D3A873468D76CDF932076F">>}, {<<"enats">>, <<"D7459C804013CAFA4AF880B18D446C48890D28D372D62AD66C76187E5779248D">>}, {<<"enats_msg">>, <<"50631124F37D88BE76A91A5B96A6565C5981EBF917CD819F0175CA658A966F43">>}, - {<<"erlcass">>, <<"17AC8F39B0A4703B12A8994663920993E191757831E125CF2C7EFB267DB66E7D">>}, {<<"ezstd">>, <<"D3B483D6ACFADFB65DBA4015371E6D54526DBF3D9EF0941B5ADD8BF5890731F4">>}, {<<"jose">>, <<"A903F5227417BD2A08C8A00A0CBCC458118BE84480955E8D251297A425723F83">>}, {<<"opentelemetry_api">>, <<"63CA1742F92F00059298F478048DFB826F4B20D49534493D6919A0DB39B6DB04">>}, @@ -32,7 +30,6 @@ {<<"cowlib">>, <<"7F478D80D66B747344F0EA7708C187645CFCC08B11AA424632F78E25BF05DB51">>}, {<<"enats">>, <<"20DEB3CB1D3E960194DF8B136C40D2DB085B485BBA5E493B340AB5F9FD2BED22">>}, {<<"enats_msg">>, <<"C4F2139E5144FABC99AFE01B8B016AD9DA278CDDC60857AC5D5AFB0AD1283534">>}, - {<<"erlcass">>, <<"37237C12ABE5745E0F900180D0ADB129634619BD6B2F57995BA020E63DDC6BEC">>}, {<<"ezstd">>, <<"28CFA0ED6CC3922095AD5BA0F23392A1664273358B17184BAA909868361184E7">>}, {<<"jose">>, <<"0D6CD36FF8BA174DB29148FC112B5842186B68A90CE9FC2B3EC3AFE76593E614">>}, {<<"opentelemetry_api">>, <<"3DFBBFAA2C2ED3121C5C483162836C4F9027DEF469C41578AF5EF32589FCFC58">>}, diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl index 81ec09b4f..64c7b6682 100644 --- a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl +++ b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl @@ -59,7 +59,7 @@ env_services_config() -> -spec env_gateway_config() -> map(). env_gateway_config() -> - maps:merge(env_gateway_base_config(), env_gateway_hotpatch_config()). + env_gateway_base_config(). -spec env_gateway_base_config() -> map(). env_gateway_base_config() -> @@ -107,36 +107,6 @@ env_gateway_base_config() -> ) }. --spec env_gateway_hotpatch_config() -> map(). -env_gateway_hotpatch_config() -> - #{ - <<"hotpatch_enabled">> => env_bool("FLUXER_GATEWAY_HOTPATCH_ENABLED", false), - <<"hotpatch_cassandra_hosts">> => env_optional_binary( - "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_HOSTS" - ), - <<"hotpatch_cassandra_port">> => env_int( - "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PORT", 9042 - ), - <<"hotpatch_cassandra_keyspace">> => env_binary( - "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_KEYSPACE", <<"fluxer">> - ), - <<"hotpatch_cassandra_username">> => env_optional_binary( - "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_USERNAME" - ), - <<"hotpatch_cassandra_password">> => env_optional_binary( - "FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PASSWORD" - ), - <<"hotpatch_public_keys">> => env_optional_binary( - "FLUXER_GATEWAY_HOTPATCH_PUBLIC_KEYS" - ), - <<"hotpatch_poll_interval_ms">> => env_int( - "FLUXER_GATEWAY_HOTPATCH_POLL_INTERVAL_MS", 5000 - ), - <<"hotpatch_startup_sync_timeout_ms">> => env_int( - "FLUXER_GATEWAY_HOTPATCH_STARTUP_SYNC_TIMEOUT_MS", 30000 - ) - }. - -spec env_nats_config() -> map(). env_nats_config() -> #{ @@ -295,20 +265,6 @@ build_http_config(Service) -> ), gateway_http_recovery_timeout_ms => get_int(Service, <<"gateway_http_recovery_timeout_ms">>, 15000), - hotpatch_enabled => get_bool(Service, <<"hotpatch_enabled">>, false), - hotpatch_cassandra_hosts => - optional_string(get_optional_binary(Service, <<"hotpatch_cassandra_hosts">>)), - hotpatch_cassandra_port => get_int(Service, <<"hotpatch_cassandra_port">>, 9042), - hotpatch_cassandra_keyspace => - get_binary(Service, <<"hotpatch_cassandra_keyspace">>, <<"fluxer">>), - hotpatch_cassandra_username => - get_optional_binary(Service, <<"hotpatch_cassandra_username">>), - hotpatch_cassandra_password => - get_optional_binary(Service, <<"hotpatch_cassandra_password">>), - hotpatch_public_keys => get_optional_binary(Service, <<"hotpatch_public_keys">>), - hotpatch_poll_interval_ms => get_int(Service, <<"hotpatch_poll_interval_ms">>, 5000), - hotpatch_startup_sync_timeout_ms => - get_int(Service, <<"hotpatch_startup_sync_timeout_ms">>, 30000), gateway_http_cleanup_interval_ms => get_int(Service, <<"gateway_http_cleanup_interval_ms">>, 30000), gateway_http_cleanup_max_age_ms => diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl index 1bea7e15d..e2b953955 100644 --- a/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl +++ b/fluxer_gateway/src/gateway/fluxer_gateway_sup.erl @@ -38,8 +38,7 @@ common_children() -> child_spec(gateway_nats_pool, gateway_nats_pool), child_spec(gateway_event_pause, gateway_event_pause), child_spec(gateway_concurrency, gateway_concurrency), - child_spec(gateway_rollout_config, gateway_rollout_config), - child_spec(gateway_hotpatch_reconciler, gateway_hotpatch_reconciler) + child_spec(gateway_rollout_config, gateway_rollout_config) ] ++ cluster_children() ++ [ child_spec(gateway_dispatch_relay, gateway_dispatch_relay), diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl deleted file mode 100644 index 401595834..000000000 --- a/fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl +++ /dev/null @@ -1,292 +0,0 @@ -%% SPDX-License-Identifier: AGPL-3.0-or-later - --module(gateway_hotpatch_bundle). --typing([eqwalizer]). - --export([ - compress_term/1, - decompress_term/1, - bundle_hash/1, - signing_payload/1, - sign/2, - verify_signature/4, - parse_public_keys/1, - decode_signed_event/2 -]). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. - --define(DOMAIN, <<"fluxer-gateway-hotpatch-v1">>). --define(ZSTD_LEVEL, 3). - --spec compress_term(term()) -> {ok, binary()} | {error, term()}. -compress_term(Term) -> - compress_binary(term_to_binary(Term, [deterministic])). - --spec decompress_term(binary()) -> {ok, term()} | {error, term()}. -decompress_term(Compressed) when is_binary(Compressed) -> - case decompress_binary(Compressed) of - {ok, Binary} -> decode_term(Binary); - {error, Reason} -> {error, Reason} - end. - --spec decode_term(binary()) -> {ok, term()} | {error, term()}. -decode_term(Binary) -> - try - {ok, binary_to_term(Binary, [safe])} - catch - Class:Reason -> {error, {invalid_term, Class, Reason}} - end. - --spec bundle_hash(binary()) -> binary(). -bundle_hash(CompressedBundle) when is_binary(CompressedBundle) -> - crypto:hash(sha256, CompressedBundle). - --spec signing_payload(binary()) -> binary(). -signing_payload(CompressedBundle) when is_binary(CompressedBundle) -> - <>. - --spec sign(binary(), binary()) -> {ok, binary()} | {error, term()}. -sign(CompressedBundle, PrivateKey) when is_binary(CompressedBundle), is_binary(PrivateKey) -> - try - {ok, crypto:sign(eddsa, none, signing_payload(CompressedBundle), [PrivateKey, ed25519])} - catch - Class:Reason -> {error, {sign_failed, Class, Reason}} - end. - --spec verify_signature(binary(), binary(), binary(), #{binary() => binary()}) -> - ok | {error, term()}. -verify_signature(CompressedBundle, SignerKeyId, Signature, PublicKeys) when - is_binary(CompressedBundle), - is_binary(SignerKeyId), - is_binary(Signature), - is_map(PublicKeys) --> - case maps:get(SignerKeyId, PublicKeys, undefined) of - undefined -> - {error, {unknown_signer, SignerKeyId}}; - PublicKey -> - verify_with_key(CompressedBundle, Signature, PublicKey) - end. - --spec parse_public_keys(term()) -> - {ok, #{binary() => binary()}} | {error, term()}. -parse_public_keys(undefined) -> - {ok, #{}}; -parse_public_keys(Value) when is_list(Value) -> - parse_public_keys(type_conv:ensure_binary(Value)); -parse_public_keys(Value) when is_binary(Value) -> - Tokens = [ - string:trim(Token) - || Token <- binary:split(Value, [<<",">>, <<"\n">>, <<";">>], [global]), - string:trim(Token) =/= <<>> - ], - parse_public_key_tokens(Tokens, #{}); -parse_public_keys(Value) -> - {error, {invalid_public_keys_config, Value}}. - --spec decode_signed_event(map(), #{binary() => binary()}) -> {ok, map()} | {error, term()}. -decode_signed_event(Event, PublicKeys) when is_map(Event), is_map(PublicKeys) -> - Bundle = event_value(bundle, Event), - SignerKeyId = event_value(signer_key_id, Event), - Signature = event_value(signature, Event), - BundleHash = event_value(bundle_sha256, Event), - decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys). - --spec decode_event_payload(term(), term(), term(), term(), #{binary() => binary()}) -> - {ok, map()} | {error, term()}. -decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys) when - is_binary(Bundle), is_binary(SignerKeyId), is_binary(Signature), is_binary(BundleHash) --> - case validate_event_hash(Bundle, BundleHash) of - ok -> decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys); - {error, Reason} -> {error, Reason} - end; -decode_event_payload(_Bundle, _SignerKeyId, _Signature, _BundleHash, _PublicKeys) -> - {error, invalid_event_payload}. - --spec decode_verified_event(binary(), binary(), binary(), #{binary() => binary()}) -> - {ok, map()} | {error, term()}. -decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys) -> - case verify_signature(Bundle, SignerKeyId, Signature, PublicKeys) of - ok -> decompress_bundle_map(Bundle); - {error, Reason} -> {error, Reason} - end. - --spec decompress_bundle_map(binary()) -> {ok, map()} | {error, term()}. -decompress_bundle_map(Bundle) -> - case decompress_term(Bundle) of - {ok, Map} when is_map(Map) -> {ok, Map}; - {ok, Other} -> {error, {invalid_bundle_term, Other}}; - {error, Reason} -> {error, Reason} - end. - --spec compress_binary(binary()) -> {ok, binary()} | {error, term()}. -compress_binary(Binary) -> - try erlang:apply(ezstd, compress, [Binary, ?ZSTD_LEVEL]) of - Compressed when is_binary(Compressed) -> {ok, Compressed}; - {error, Reason} -> {error, {compress_failed, Reason}}; - Other -> {error, {compress_failed, Other}} - catch - Class:Reason -> {error, {compress_failed, Class, Reason}} - end. - --spec decompress_binary(binary()) -> {ok, binary()} | {error, term()}. -decompress_binary(Binary) -> - try erlang:apply(ezstd, decompress, [Binary]) of - Decompressed when is_binary(Decompressed) -> {ok, Decompressed}; - Decompressed when is_list(Decompressed) -> {ok, iolist_to_binary(Decompressed)}; - {error, Reason} -> {error, {decompress_failed, Reason}}; - Other -> {error, {decompress_failed, Other}} - catch - Class:Reason -> {error, {decompress_failed, Class, Reason}} - end. - --spec verify_with_key(binary(), binary(), binary()) -> ok | {error, term()}. -verify_with_key(CompressedBundle, Signature, PublicKey) when byte_size(PublicKey) =:= 32 -> - verify_with_valid_key(CompressedBundle, Signature, PublicKey); -verify_with_key(_CompressedBundle, _Signature, PublicKey) -> - {error, {invalid_public_key_size, byte_size(PublicKey)}}. - --spec verify_with_valid_key(binary(), binary(), binary()) -> ok | {error, term()}. -verify_with_valid_key(CompressedBundle, Signature, PublicKey) -> - try - crypto:verify(eddsa, none, signing_payload(CompressedBundle), Signature, [ - PublicKey, ed25519 - ]) - of - true -> ok; - false -> {error, invalid_signature} - catch - Class:Reason -> {error, {verify_failed, Class, Reason}} - end. - --spec parse_public_key_tokens([binary()], #{binary() => binary()}) -> - {ok, #{binary() => binary()}} | {error, term()}. -parse_public_key_tokens([], Acc) -> - {ok, Acc}; -parse_public_key_tokens([Token | Rest], Acc) -> - case parse_public_key_token(Token) of - {ok, KeyId, PublicKey} -> parse_public_key_tokens(Rest, Acc#{KeyId => PublicKey}); - {error, Reason} -> {error, Reason} - end. - --spec parse_public_key_token(binary()) -> {ok, binary(), binary()} | {error, term()}. -parse_public_key_token(Token) -> - case split_key_token(Token) of - {ok, KeyId, Encoded} -> parse_public_key_material(KeyId, Encoded); - error -> {error, {invalid_public_key_token, Token}} - end. - --spec parse_public_key_material(binary(), binary()) -> - {ok, binary(), binary()} | {error, term()}. -parse_public_key_material(KeyId, Encoded) -> - case decode_key_material(Encoded) of - {ok, PublicKey} when byte_size(PublicKey) =:= 32 -> {ok, KeyId, PublicKey}; - {ok, PublicKey} -> {error, {invalid_public_key_size, KeyId, byte_size(PublicKey)}}; - {error, Reason} -> {error, {invalid_public_key, KeyId, Reason}} - end. - --spec split_key_token(binary()) -> {ok, binary(), binary()} | error. -split_key_token(Token) -> - case binary:split(Token, <<":">>) of - [KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)}; - _ -> split_key_token_equals(Token) - end. - --spec split_key_token_equals(binary()) -> {ok, binary(), binary()} | error. -split_key_token_equals(Token) -> - case binary:split(Token, <<"=">>) of - [KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)}; - _ -> error - end. - --spec decode_key_material(binary()) -> {ok, binary()} | {error, term()}. -decode_key_material(Encoded) -> - case try_base64(Encoded) of - {ok, Decoded} -> {ok, Decoded}; - {error, _} -> try_base64url(Encoded) - end. - --spec try_base64(binary()) -> {ok, binary()} | {error, term()}. -try_base64(Encoded) -> - try - {ok, base64:decode(Encoded)} - catch - Class:Reason -> {error, {Class, Reason}} - end. - --spec try_base64url(binary()) -> {ok, binary()} | {error, term()}. -try_base64url(Encoded) -> - try - {ok, base64url:decode(Encoded)} - catch - Class:Reason -> {error, {Class, Reason}} - end. - --spec validate_event_hash(binary(), binary()) -> ok | {error, term()}. -validate_event_hash(Bundle, BundleHash) -> - case bundle_hash(Bundle) of - BundleHash -> ok; - Other -> {error, {bundle_hash_mismatch, Other, BundleHash}} - end. - --spec event_value(atom(), map()) -> term(). -event_value(Key, Event) -> - maps:get(Key, Event, maps:get(atom_to_binary(Key, utf8), Event, undefined)). - --ifdef(TEST). - -compress_decompress_roundtrip_test() -> - Term = #{ - version => 1, - build_sha => <<"abc123">>, - modules => [#{module => <<"session_lifecycle">>, expected_current_md5 => <<0:128>>}] - }, - {ok, Compressed} = compress_term(Term), - ?assert(is_binary(Compressed)), - ?assertEqual({ok, Term}, decompress_term(Compressed)). - -sign_and_verify_roundtrip_test() -> - {PublicKey, PrivateKey} = ed25519_keypair(), - {ok, Compressed} = compress_term(#{version => 1}), - {ok, Signature} = sign(Compressed, PrivateKey), - Keys = #{<<"ops">> => PublicKey}, - ?assertEqual(ok, verify_signature(Compressed, <<"ops">>, Signature, Keys)), - ?assertEqual( - {error, invalid_signature}, - verify_signature(<>, <<"ops">>, Signature, Keys) - ). - -decode_signed_event_rejects_hash_mismatch_test() -> - {PublicKey, PrivateKey} = ed25519_keypair(), - {ok, Compressed} = compress_term(#{version => 1}), - {ok, Signature} = sign(Compressed, PrivateKey), - Event = #{ - signer_key_id => <<"ops">>, - signature => Signature, - bundle_sha256 => <<0:256>>, - bundle => Compressed - }, - ?assertMatch( - {error, {bundle_hash_mismatch, _, _}}, - decode_signed_event(Event, #{<<"ops">> => PublicKey}) - ). - -parse_public_keys_test() -> - PublicKey = <<1:256>>, - Encoded = base64:encode(PublicKey), - ?assertEqual( - {ok, #{<<"ops">> => PublicKey}}, parse_public_keys(<<"ops:", Encoded/binary>>) - ). - -ed25519_keypair() -> - {PublicKey, PrivateKey} = crypto:generate_key(eddsa, ed25519), - {require_binary(PublicKey), require_binary(PrivateKey)}. - -require_binary(Value) when is_binary(Value) -> - Value. - --endif. diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl deleted file mode 100644 index df8f13c5b..000000000 --- a/fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl +++ /dev/null @@ -1,459 +0,0 @@ -%% SPDX-License-Identifier: AGPL-3.0-or-later - --module(gateway_hotpatch_cli). --typing([eqwalizer]). - --export([main/1, build_bundle/2, sign_bundle/3]). - --spec main([string()]) -> no_return(). -main(Args) -> - halt(run_main(Args)). - --spec run_main([string()]) -> non_neg_integer(). -run_main(["bundle", BuildSha | ModuleOrder]) when ModuleOrder =/= [] -> - write_json_result(build_bundle(type_conv:ensure_binary(BuildSha), ModuleOrder)); -run_main(["sign", SignerKeyId, PrivateKeyPath, BundlePath]) -> - write_json_result(sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath)); -run_main(["append", BuildSha, CreatedBy, EventPath]) -> - Result = append_event( - type_conv:ensure_binary(BuildSha), type_conv:ensure_binary(CreatedBy), EventPath - ), - write_append_result(Result); -run_main(_Args) -> - write_usage(), - 64. - --spec build_bundle(binary(), [string()]) -> {ok, map()} | {error, term()}. -build_bundle(BuildSha, ModuleSpecs) when is_binary(BuildSha), is_list(ModuleSpecs) -> - build_bundle_modules(ModuleSpecs, fun(Modules) -> - {ok, #{<<"version">> => 1, <<"build_sha">> => BuildSha, <<"modules">> => Modules}} - end). - --spec sign_bundle(binary(), file:filename(), binary()) -> {ok, map()} | {error, term()}. -sign_bundle(SignerKeyId, PrivateKeyPath, BundleJson) when - is_binary(SignerKeyId), is_binary(BundleJson) --> - case decode_json(BundleJson) of - {ok, Bundle} -> sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle); - {error, Reason} -> {error, Reason} - end. - --spec sign_decoded_bundle_with_key(binary(), file:filename(), term()) -> - {ok, map()} | {error, term()}. -sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle) -> - case read_private_key(PrivateKeyPath) of - {ok, PrivateKey} -> sign_decoded_bundle(SignerKeyId, Bundle, PrivateKey); - {error, Reason} -> {error, Reason} - end. - --spec sign_bundle_file(string(), file:filename(), file:filename()) -> - {ok, map()} | {error, term()}. -sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath) -> - case file:read_file(BundlePath) of - {ok, BundleJson} -> - sign_bundle(type_conv:ensure_binary(SignerKeyId), PrivateKeyPath, BundleJson); - {error, Reason} -> - {error, {read_bundle_failed, Reason}} - end. - --spec sign_decoded_bundle(binary(), term(), binary()) -> {ok, map()} | {error, term()}. -sign_decoded_bundle(SignerKeyId, BundleJson, PrivateKey) -> - case json_to_bundle(BundleJson) of - {ok, Bundle} -> sign_bundle_term(SignerKeyId, Bundle, PrivateKey); - {error, Reason} -> {error, Reason} - end. - --spec sign_bundle_term(binary(), map(), binary()) -> {ok, map()} | {error, term()}. -sign_bundle_term(SignerKeyId, Bundle, PrivateKey) -> - case gateway_hotpatch_bundle:compress_term(Bundle) of - {ok, Compressed} -> sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey); - {error, Reason} -> {error, Reason} - end. - --spec sign_compressed_bundle(binary(), binary(), binary()) -> {ok, map()} | {error, term()}. -sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey) -> - case gateway_hotpatch_bundle:sign(Compressed, PrivateKey) of - {ok, Signature} -> {ok, signed_event(SignerKeyId, Compressed, Signature)}; - {error, Reason} -> {error, Reason} - end. - --spec signed_event(binary(), binary(), binary()) -> map(). -signed_event(SignerKeyId, Compressed, Signature) -> - #{ - <<"schema_version">> => 1, - <<"kind">> => <<"beam_bundle">>, - <<"created_by">> => gateway_hotpatch_runtime:node_name(), - <<"signer_key_id">> => SignerKeyId, - <<"bundle_sha256">> => encode_bytes(gateway_hotpatch_bundle:bundle_hash(Compressed)), - <<"signature">> => encode_bytes(Signature), - <<"bundle">> => encode_bytes(Compressed) - }. - --spec append_event(binary(), binary(), file:filename()) -> {ok, binary()} | {error, term()}. -append_event(BuildSha, CreatedBy, EventPath) -> - case file:read_file(EventPath) of - {ok, EventJson} -> append_event_json(BuildSha, CreatedBy, EventJson); - {error, Reason} -> {error, {read_event_failed, Reason}} - end. - --spec append_event_json(binary(), binary(), binary()) -> {ok, binary()} | {error, term()}. -append_event_json(BuildSha, CreatedBy, EventJson) -> - case decode_json(EventJson) of - {ok, EventJsonTerm} -> append_event_term(BuildSha, CreatedBy, EventJsonTerm); - {error, Reason} -> {error, Reason} - end. - --spec append_event_term(binary(), binary(), term()) -> {ok, binary()} | {error, term()}. -append_event_term(BuildSha, CreatedBy, EventJsonTerm) -> - case json_to_event(EventJsonTerm) of - {ok, Event} -> append_event_row(BuildSha, CreatedBy, Event); - {error, Reason} -> {error, Reason} - end. - --spec build_bundle_modules([string()], fun(([map()]) -> {ok, map()})) -> - {ok, map()} | {error, term()}. -build_bundle_modules(ModuleSpecs, Fun) -> - case build_bundle_module_list(ModuleSpecs, []) of - {ok, Modules} -> Fun(Modules); - {error, Reason} -> {error, Reason} - end. - --spec build_bundle_module_list([string()], [map()]) -> {ok, [map()]} | {error, term()}. -build_bundle_module_list([], Acc) -> - {ok, lists:reverse(Acc)}; -build_bundle_module_list([ModuleSpec | Rest], Acc) -> - case bundle_module(ModuleSpec) of - {ok, Module} -> build_bundle_module_list(Rest, [Module | Acc]); - {error, Reason} -> {error, Reason} - end. - --spec bundle_module(string()) -> {ok, map()} | {error, term()}. -bundle_module(ModuleSpec) -> - case split_module_spec(ModuleSpec) of - {ok, ModuleName, BeamPath0} -> bundle_named_module(ModuleName, BeamPath0); - {error, Reason} -> {error, Reason} - end. - --spec bundle_named_module(string(), file:filename() | undefined) -> - {ok, map()} | {error, term()}. -bundle_named_module(ModuleName, BeamPath0) -> - case existing_module(ModuleName) of - {ok, Module} -> bundle_existing_module(Module, BeamPath0); - {error, Reason} -> {error, Reason} - end. - --spec bundle_existing_module(atom(), file:filename() | undefined) -> - {ok, map()} | {error, term()}. -bundle_existing_module(Module, BeamPath0) -> - case collect_bundle_parts(Module, BeamPath0) of - {ok, CurrentMd5, TargetMd5, CompressedBeam} -> - {ok, build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam)}; - {error, Reason} -> - {error, Reason} - end. - --spec collect_bundle_parts(atom(), file:filename() | undefined) -> - {ok, binary(), binary(), binary()} | {error, term()}. -collect_bundle_parts(Module, BeamPath0) -> - case gateway_hotpatch_loader:current_md5(Module) of - {ok, CurrentMd5} -> collect_bundle_beam(Module, BeamPath0, CurrentMd5); - {error, Reason} -> {error, {current_md5_failed, Module, Reason}} - end. - --spec collect_bundle_beam(atom(), file:filename() | undefined, binary()) -> - {ok, binary(), binary(), binary()} | {error, term()}. -collect_bundle_beam(Module, BeamPath0, CurrentMd5) -> - case resolve_beam_path(Module, BeamPath0) of - {ok, BeamPath} -> read_bundle_beam(Module, BeamPath, CurrentMd5); - {error, Reason} -> {error, Reason} - end. - --spec read_bundle_beam(atom(), file:filename(), binary()) -> - {ok, binary(), binary(), binary()} | {error, term()}. -read_bundle_beam(Module, BeamPath, CurrentMd5) -> - case file:read_file(BeamPath) of - {ok, Beam} -> hash_bundle_beam(Module, CurrentMd5, Beam); - {error, Reason} -> {error, {read_beam_failed, Module, BeamPath, Reason}} - end. - --spec hash_bundle_beam(atom(), binary(), binary()) -> - {ok, binary(), binary(), binary()} | {error, term()}. -hash_bundle_beam(Module, CurrentMd5, Beam) -> - case gateway_hotpatch_loader:beam_md5(Beam) of - {ok, TargetMd5} -> compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam); - {error, Reason} -> {error, {target_md5_failed, Module, Reason}} - end. - --spec compress_bundle_beam(atom(), binary(), binary(), binary()) -> - {ok, binary(), binary(), binary()} | {error, term()}. -compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam) -> - case compress_beam(Beam) of - {ok, CompressedBeam} -> {ok, CurrentMd5, TargetMd5, CompressedBeam}; - {error, Reason} -> {error, {beam_compress_failed, Module, Reason}} - end. - --spec build_module_entry(atom(), binary(), binary(), binary()) -> map(). -build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam) -> - #{ - <<"module">> => atom_to_binary(Module, utf8), - <<"expected_current_md5">> => encode_bytes(CurrentMd5), - <<"target_md5">> => encode_bytes(TargetMd5), - <<"beam_zstd">> => encode_bytes(CompressedBeam) - }. - --spec append_event_row(binary(), binary(), map()) -> {ok, binary()} | {error, term()}. -append_event_row(BuildSha, CreatedBy, Event) -> - _ = fluxer_gateway_env:load(), - case gateway_hotpatch_store:connect() of - ok -> - gateway_hotpatch_store:append_event( - BuildSha, - CreatedBy, - maps:get(signer_key_id, Event), - maps:get(signature, Event), - maps:get(bundle_sha256, Event), - maps:get(bundle, Event) - ); - {error, Reason} -> - {error, Reason} - end. - --spec split_module_spec(string()) -> - {ok, string(), file:filename() | undefined} | {error, term()}. -split_module_spec(ModuleSpec) -> - case string:split(ModuleSpec, "=", leading) of - [ModuleName, BeamPath] when ModuleName =/= "", BeamPath =/= "" -> - {ok, ModuleName, BeamPath}; - [ModuleName] when ModuleName =/= "" -> - {ok, ModuleName, undefined}; - _ -> - {error, {invalid_module_spec, ModuleSpec}} - end. - --spec existing_module(string()) -> {ok, atom()} | {error, term()}. -existing_module(ModuleName) -> - try - {ok, list_to_existing_atom(ModuleName)} - catch - error:badarg -> {error, {unknown_module, ModuleName}} - end. - --spec resolve_beam_path(atom(), file:filename() | undefined) -> - {ok, file:filename()} | {error, term()}. -resolve_beam_path(Module, undefined) -> - case code:which(Module) of - File when is_list(File) -> {ok, File}; - Other -> {error, {module_beam_not_found, Module, Other}} - end; -resolve_beam_path(_Module, BeamPath) -> - {ok, BeamPath}. - --spec compress_beam(binary()) -> {ok, binary()} | {error, term()}. -compress_beam(Beam) -> - try erlang:apply(ezstd, compress, [Beam, 3]) of - Compressed when is_binary(Compressed) -> {ok, Compressed}; - {error, Reason} -> {error, Reason}; - Other -> {error, Other} - catch - Class:Reason -> {error, {Class, Reason}} - end. - --spec decode_json(binary()) -> {ok, term()} | {error, term()}. -decode_json(Json) -> - try - {ok, json:decode(Json)} - catch - Class:Reason -> {error, {decode_json_failed, Class, Reason}} - end. - --spec read_private_key(file:filename()) -> {ok, binary()} | {error, term()}. -read_private_key(Path) -> - case file:read_file(Path) of - {ok, PrivateKey} when byte_size(PrivateKey) =:= 32 -> - {ok, PrivateKey}; - {ok, PrivateKey} -> - {error, {invalid_private_key_size, byte_size(PrivateKey)}}; - {error, Reason} -> - {error, {read_private_key_failed, Reason}} - end. - --spec json_to_bundle(term()) -> {ok, map()} | {error, term()}. -json_to_bundle(#{ - <<"version">> := Version, <<"build_sha">> := BuildSha, <<"modules">> := Modules -}) when is_list(Modules) -> - case json_to_module_entries(Modules, []) of - {ok, Entries} -> - {ok, #{ - <<"version">> => Version, - <<"build_sha">> => BuildSha, - <<"modules">> => Entries - }}; - {error, Reason} -> - {error, Reason} - end; -json_to_bundle(Other) -> - {error, {invalid_bundle_json, Other}}. - --spec json_to_module_entries([term()], [map()]) -> {ok, [map()]} | {error, term()}. -json_to_module_entries([], Acc) -> - {ok, lists:reverse(Acc)}; -json_to_module_entries([Module | Rest], Acc) -> - case json_to_module_entry(Module) of - {ok, Entry} -> json_to_module_entries(Rest, [Entry | Acc]); - {error, Reason} -> {error, Reason} - end. - --spec json_to_module_entry(term()) -> {ok, map()} | {error, term()}. -json_to_module_entry(#{ - <<"module">> := Module, - <<"expected_current_md5">> := ExpectedMd5, - <<"target_md5">> := TargetMd5, - <<"beam_zstd">> := BeamZstd -}) -> - decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd); -json_to_module_entry(Other) -> - {error, {invalid_module_entry_json, Other}}. - --spec decode_module_entry(term(), term(), term(), term()) -> - {ok, map()} | {error, term()}. -decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) when - is_binary(Module), is_binary(ExpectedMd5), is_binary(TargetMd5), is_binary(BeamZstd) --> - case decode_bytes(ExpectedMd5) of - {ok, ExpectedMd5Bytes} -> - decode_module_entry_target(Module, ExpectedMd5Bytes, TargetMd5, BeamZstd); - {error, Reason} -> - {error, Reason} - end; -decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) -> - {error, {invalid_module_entry_json, {Module, ExpectedMd5, TargetMd5, BeamZstd}}}. - --spec decode_module_entry_target(binary(), binary(), binary(), binary()) -> - {ok, map()} | {error, term()}. -decode_module_entry_target(Module, ExpectedMd5, TargetMd5, BeamZstd) -> - case decode_bytes(TargetMd5) of - {ok, TargetMd5Bytes} -> - decode_module_entry_beam(Module, ExpectedMd5, TargetMd5Bytes, BeamZstd); - {error, Reason} -> - {error, Reason} - end. - --spec decode_module_entry_beam(binary(), binary(), binary(), binary()) -> - {ok, map()} | {error, term()}. -decode_module_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd) -> - case decode_bytes(BeamZstd) of - {ok, BeamZstdBytes} -> - {ok, #{ - <<"module">> => Module, - <<"expected_current_md5">> => ExpectedMd5, - <<"target_md5">> => TargetMd5, - <<"beam_zstd">> => BeamZstdBytes - }}; - {error, Reason} -> - {error, Reason} - end. - --spec json_to_event(term()) -> {ok, map()} | {error, term()}. -json_to_event(#{ - <<"signer_key_id">> := SignerKeyId, - <<"bundle_sha256">> := BundleSha256, - <<"signature">> := Signature, - <<"bundle">> := Bundle -}) -> - decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle); -json_to_event(Other) -> - {error, {invalid_event_json, Other}}. - --spec decode_event_bytes(term(), term(), term(), term()) -> - {ok, map()} | {error, term()}. -decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) when - is_binary(SignerKeyId), is_binary(BundleSha256), is_binary(Signature), is_binary(Bundle) --> - case decode_bytes(BundleSha256) of - {ok, BundleSha256Bytes} -> - decode_event_signature(SignerKeyId, BundleSha256Bytes, Signature, Bundle); - {error, Reason} -> - {error, Reason} - end; -decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) -> - {error, {invalid_event_json, {SignerKeyId, BundleSha256, Signature, Bundle}}}. - --spec decode_event_signature(binary(), binary(), binary(), binary()) -> - {ok, map()} | {error, term()}. -decode_event_signature(SignerKeyId, BundleSha256, Signature, Bundle) -> - case decode_bytes(Signature) of - {ok, SignatureBytes} -> - decode_event_bundle(SignerKeyId, BundleSha256, SignatureBytes, Bundle); - {error, Reason} -> - {error, Reason} - end. - --spec decode_event_bundle(binary(), binary(), binary(), binary()) -> - {ok, map()} | {error, term()}. -decode_event_bundle(SignerKeyId, BundleSha256, Signature, Bundle) -> - case decode_bytes(Bundle) of - {ok, BundleBytes} -> - {ok, #{ - signer_key_id => SignerKeyId, - bundle_sha256 => BundleSha256, - signature => Signature, - bundle => BundleBytes - }}; - {error, Reason} -> - {error, Reason} - end. - --spec encode_bytes(binary()) -> binary(). -encode_bytes(Binary) -> - base64:encode(Binary). - --spec decode_bytes(binary()) -> {ok, binary()} | {error, term()}. -decode_bytes(Encoded) when is_binary(Encoded) -> - try - {ok, base64:decode(Encoded)} - catch - Class:Reason -> {error, {invalid_base64_bytes, Encoded, Class, Reason}} - end. - --spec write_json_result({ok, map()} | {error, term()}) -> non_neg_integer(). -write_json_result({ok, Term}) -> - write_stdout("~ts~n", [json:encode(Term)]), - 0; -write_json_result({error, Reason}) -> - write_error(Reason). - --spec write_append_result({ok, binary()} | {error, term()}) -> non_neg_integer(). -write_append_result({ok, EventId}) -> - write_stdout("appended hotpatch event ~ts~n", [gateway_hotpatch_loader:hex(EventId)]), - 0; -write_append_result({error, Reason}) -> - write_error(Reason). - --spec write_error(term()) -> non_neg_integer(). -write_error(Reason) -> - write_stderr("gateway hotpatch failed: ~0tp~n", [Reason]), - 1. - --spec write_usage() -> ok. -write_usage() -> - write_stderr( - "usage: gateway_hotpatch_cli bundle BUILD_SHA module_a=/path/to/module_a.beam ...~n" - " gateway_hotpatch_cli sign SIGNER_KEY_ID PRIVATE_KEY_RAW_FILE bundle.json~n" - " gateway_hotpatch_cli append BUILD_SHA CREATED_BY signed-event.json~n", - [] - ). - --spec write_stdout(io:format(), [term()]) -> ok. -write_stdout(Format, Args) -> - write_stream(standard_io, Format, Args). - --spec write_stderr(io:format(), [term()]) -> ok. -write_stderr(Format, Args) -> - write_stream(standard_error, Format, Args). - --spec write_stream(file:io_device() | standard_io | standard_error, io:format(), [term()]) -> - ok. -write_stream(Device, Format, Args) -> - Output = iolist_to_binary(io_lib:format(Format, Args)), - _ = file:write(Device, Output), - ok. diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl deleted file mode 100644 index c6b81d49d..000000000 --- a/fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl +++ /dev/null @@ -1,433 +0,0 @@ -%% SPDX-License-Identifier: AGPL-3.0-or-later - --module(gateway_hotpatch_loader). --typing([eqwalizer]). - --export([ - apply_bundle/1, - current_md5/1, - beam_module/1, - beam_md5/1, - hex/1 -]). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. - --type module_entry() :: map(). --type apply_result() :: {ok, applied | skipped, atom(), binary()} | {error, term()}. - --spec apply_bundle(map()) -> {ok, map()} | {error, term()}. -apply_bundle(Bundle) when is_map(Bundle) -> - case - { - entry_value(version, Bundle), - entry_value(build_sha, Bundle), - entry_value(modules, Bundle) - } - of - {1, BundleBuildSha, Modules} when is_list(Modules) -> - apply_versioned_bundle(BundleBuildSha, Modules); - _ -> - {error, invalid_bundle} - end; -apply_bundle(_Bundle) -> - {error, invalid_bundle}. - --spec apply_versioned_bundle(term(), [module_entry()]) -> {ok, map()} | {error, term()}. -apply_versioned_bundle(BundleBuildSha, Modules) -> - CurrentBuildSha = gateway_hotpatch_runtime:build_sha(), - case normalize_binary(BundleBuildSha) of - CurrentBuildSha -> apply_modules(Modules, #{applied => [], skipped => []}); - Other -> {error, {build_sha_mismatch, Other, CurrentBuildSha}} - end. - --spec current_md5(atom()) -> {ok, binary()} | {error, term()}. -current_md5(Module) when is_atom(Module) -> - case hotpatch_loaded_md5(Module) of - {ok, Md5} -> {ok, Md5}; - error -> current_md5_result(Module, current_md5_from_loaded_or_file(Module)) - end. - --spec current_md5_result(atom(), {ok, binary()} | {error, term()}) -> - {ok, binary()} | {error, term()}. -current_md5_result(_Module, {ok, Md5}) -> - {ok, Md5}; -current_md5_result(Module, {error, Reason}) -> - fallback_current_md5(Module, Reason). - --spec fallback_current_md5(atom(), term()) -> {ok, binary()} | {error, term()}. -fallback_current_md5(Module, Reason) -> - case hotpatch_loaded_md5(Module) of - {ok, Md5} -> {ok, Md5}; - error -> {error, Reason} - end. - --spec current_md5_from_loaded_or_file(atom()) -> {ok, binary()} | {error, term()}. -current_md5_from_loaded_or_file(Module) -> - case code:get_object_code(Module) of - {Module, Beam, _File} when is_binary(Beam) -> beam_md5(Beam); - error -> current_md5_from_file(Module) - end. - --spec beam_module(binary()) -> {ok, atom()} | {error, term()}. -beam_module(Beam) when is_binary(Beam) -> - case beam_lib:info(Beam) of - Info when is_list(Info) -> beam_module_from_info(Info); - Other -> {error, {beam_info_failed, Other}} - end. - --spec beam_module_from_info(list()) -> {ok, atom()} | {error, term()}. -beam_module_from_info(Info) -> - case lists:keyfind(module, 1, Info) of - {module, Module} when is_atom(Module) -> {ok, Module}; - _ -> {error, module_not_found} - end. - --spec beam_md5(binary()) -> {ok, binary()} | {error, term()}. -beam_md5(Beam) when is_binary(Beam) -> - case beam_lib:md5(Beam) of - {ok, {_Module, Md5}} when is_binary(Md5) -> {ok, Md5}; - Error -> {error, {beam_md5_failed, Error}} - end. - --spec hex(binary()) -> binary(). -hex(Binary) when is_binary(Binary) -> - iolist_to_binary([[hex_nibble(High), hex_nibble(Low)] || <> <= Binary]). - --spec apply_modules([module_entry()], map()) -> {ok, map()} | {error, term()}. -apply_modules([], Acc) -> - {ok, Acc#{module_count => length(maps:get(applied, Acc)) + length(maps:get(skipped, Acc))}}; -apply_modules([Entry | Rest], Acc) -> - case apply_module(Entry) of - {ok, applied, Module, TargetMd5} -> - apply_modules( - Rest, - Acc#{applied => [{Module, hex(TargetMd5)} | maps:get(applied, Acc)]} - ); - {ok, skipped, Module, TargetMd5} -> - apply_modules( - Rest, - Acc#{skipped => [{Module, hex(TargetMd5)} | maps:get(skipped, Acc)]} - ); - {error, Reason} -> - {error, Reason} - end. - --spec apply_module(module_entry()) -> apply_result(). -apply_module(Entry) -> - with_entry(Entry, fun apply_valid_entry/4). - --spec apply_valid_entry(atom(), binary(), binary(), binary()) -> apply_result(). -apply_valid_entry(Module, ExpectedMd5, TargetMd5, Beam) -> - case current_md5(Module) of - {ok, TargetMd5} -> - {ok, skipped, Module, TargetMd5}; - {ok, ExpectedMd5} -> - load_module(Module, Beam, TargetMd5); - {ok, CurrentMd5} -> - {error, {md5_mismatch, Module, hex(CurrentMd5), hex(ExpectedMd5)}}; - {error, Reason} -> - {error, {current_md5_failed, Module, Reason}} - end. - --spec with_entry(module_entry(), fun((atom(), binary(), binary(), binary()) -> apply_result())) -> - apply_result(). -with_entry(Entry, Fun) when is_map(Entry), is_function(Fun, 4) -> - case normalize_module(entry_value(module, Entry)) of - {ok, Module} -> - ExpectedMd5 = entry_value(expected_current_md5, Entry), - TargetMd5 = entry_value(target_md5, Entry), - BeamZstd = entry_value(beam_zstd, Entry), - with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun); - {error, Reason} -> - {error, Reason} - end; -with_entry(_Entry, _Fun) -> - {error, invalid_module_entry}. - --spec with_entry_beam(atom(), term(), term(), term(), fun( - (atom(), binary(), binary(), binary()) -> apply_result() -)) -> - apply_result(). -with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun) when - is_binary(ExpectedMd5), - is_binary(TargetMd5), - byte_size(ExpectedMd5) =:= 16, - byte_size(TargetMd5) =:= 16, - is_binary(BeamZstd) --> - case decompress_beam(BeamZstd) of - {ok, Beam} -> validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun); - {error, Reason} -> {error, {beam_decompress_failed, Module, Reason}} - end; -with_entry_beam(Module, _ExpectedMd5, _TargetMd5, _BeamZstd, _Fun) -> - {error, {invalid_module_entry, Module}}. - --spec validate_beam(atom(), binary(), binary(), binary(), fun( - (atom(), binary(), binary(), binary()) -> apply_result() -)) -> - apply_result(). -validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun) -> - case beam_module(Beam) of - {ok, Module} -> validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun); - {ok, OtherModule} -> {error, {beam_module_mismatch, Module, OtherModule}}; - {error, Reason} -> {error, {beam_module_failed, Module, Reason}} - end. - --spec validate_beam_md5(atom(), binary(), binary(), binary(), fun( - (atom(), binary(), binary(), binary()) -> apply_result() -)) -> - apply_result(). -validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun) -> - case beam_md5(Beam) of - {ok, TargetMd5} -> - Fun(Module, ExpectedMd5, TargetMd5, Beam); - {ok, OtherMd5} -> - {error, {target_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}}; - {error, Reason} -> - {error, {target_md5_failed, Module, Reason}} - end. - --spec load_module(atom(), binary(), binary()) -> apply_result(). -load_module(Module, Beam, TargetMd5) -> - case code:soft_purge(Module) of - true -> load_purged_module(Module, Beam, TargetMd5); - false -> {error, {soft_purge_failed, Module}} - end. - --spec load_purged_module(atom(), binary(), binary()) -> apply_result(). -load_purged_module(Module, Beam, TargetMd5) -> - case code:load_binary(Module, atom_to_list(Module) ++ ".beam", Beam) of - {module, Module} -> - put_hotpatch_loaded_md5(Module, TargetMd5), - verify_loaded_module(Module, TargetMd5); - {error, Reason} -> - {error, {load_binary_failed, Module, Reason}} - end. - --spec verify_loaded_module(atom(), binary()) -> apply_result(). -verify_loaded_module(Module, TargetMd5) -> - case current_md5(Module) of - {ok, TargetMd5} -> - {ok, applied, Module, TargetMd5}; - {ok, OtherMd5} -> - {error, {post_load_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}}; - {error, Reason} -> - {error, {post_load_md5_failed, Module, Reason}} - end. - --spec current_md5_from_file(atom()) -> {ok, binary()} | {error, term()}. -current_md5_from_file(Module) -> - case code:which(Module) of - File when is_list(File) -> current_md5_from_path(Module, File); - preloaded -> {error, preloaded}; - non_existing -> {error, not_loaded}; - Other -> {error, {not_loadable, Other}} - end. - --spec current_md5_from_path(atom(), file:filename()) -> {ok, binary()} | {error, term()}. -current_md5_from_path(Module, File) -> - case beam_lib:md5(File) of - {ok, {Module, Md5}} when is_binary(Md5) -> {ok, Md5}; - Error -> {error, {beam_file_md5_failed, Error}} - end. - --spec hotpatch_loaded_md5(atom()) -> {ok, binary()} | error. -hotpatch_loaded_md5(Module) -> - case persistent_term:get({?MODULE, loaded_md5, Module}, undefined) of - Md5 when is_binary(Md5), byte_size(Md5) =:= 16 -> {ok, Md5}; - _ -> error - end. - --spec put_hotpatch_loaded_md5(atom(), binary()) -> ok. -put_hotpatch_loaded_md5(Module, Md5) -> - persistent_term:put({?MODULE, loaded_md5, Module}, Md5). - --spec decompress_beam(binary()) -> {ok, binary()} | {error, term()}. -decompress_beam(Compressed) -> - try erlang:apply(ezstd, decompress, [Compressed]) of - Beam when is_binary(Beam) -> {ok, Beam}; - Beam when is_list(Beam) -> {ok, iolist_to_binary(Beam)}; - {error, Reason} -> {error, Reason}; - Other -> {error, Other} - catch - Class:Reason -> {error, {Class, Reason}} - end. - --spec normalize_module(term()) -> {ok, atom()} | {error, term()}. -normalize_module(Module) when is_atom(Module) -> - {ok, Module}; -normalize_module(Module) when is_binary(Module) -> - try - {ok, binary_to_existing_atom(Module, utf8)} - catch - error:badarg -> {error, {unknown_module, Module}} - end; -normalize_module(Module) when is_list(Module) -> - normalize_module(type_conv:ensure_binary(Module)); -normalize_module(Module) -> - {error, {invalid_module, Module}}. - --spec normalize_binary(term()) -> binary(). -normalize_binary(Bin) when is_binary(Bin) -> Bin; -normalize_binary(List) when is_list(List) -> type_conv:ensure_binary(List); -normalize_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -normalize_binary(Other) -> term_to_binary(Other). - --spec entry_value(atom(), map()) -> term(). -entry_value(Key, Entry) -> - maps:get(Key, Entry, maps:get(atom_to_binary(Key, utf8), Entry, undefined)). - --spec hex_nibble(0..15) -> integer(). -hex_nibble(N) when N < 10 -> $0 + N; -hex_nibble(N) -> $a + (N - 10). - --ifdef(TEST). - -beam_module_and_md5_test() -> - {module, ?MODULE} = code:ensure_loaded(?MODULE), - {?MODULE, Binary, _File} = code:get_object_code(?MODULE), - ?assertEqual({ok, ?MODULE}, beam_module(Binary)), - {ok, Md5} = beam_md5(Binary), - ?assertEqual(16, byte_size(Md5)). - -apply_bundle_rejects_build_mismatch_test() -> - Bundle = #{version => 1, build_sha => <<"definitely-not-this-build">>, modules => []}, - ?assertMatch({error, {build_sha_mismatch, _, _}}, apply_bundle(Bundle)). - -apply_bundle_accepts_binary_keys_test() -> - Bundle = #{ - <<"version">> => 1, - <<"build_sha">> => gateway_hotpatch_runtime:build_sha(), - <<"modules">> => [] - }, - ?assertEqual( - {ok, #{applied => [], skipped => [], module_count => 0}}, apply_bundle(Bundle) - ). - -apply_bundle_loads_new_beam_test() -> - Module = gateway_hotpatch_loader_test_target, - cleanup_test_module(Module), - try - Beam1 = compile_test_module(Module, 1), - Beam2 = compile_test_module(Module, 2), - {module, Module} = load_test_beam(Module, Beam1), - ?assertEqual(1, erlang:apply(Module, version, [])), - {ok, ExpectedMd5} = current_md5(Module), - {Entry, TargetMd5} = beam_entry(Module, ExpectedMd5, Beam2), - Bundle = #{ - version => 1, - build_sha => gateway_hotpatch_runtime:build_sha(), - modules => [Entry] - }, - ?assertEqual( - {ok, #{applied => [{Module, hex(TargetMd5)}], skipped => [], module_count => 1}}, - apply_bundle(Bundle) - ), - ?assertEqual(2, erlang:apply(Module, version, [])) - after - cleanup_test_module(Module) - end. - -apply_bundle_leaves_prior_module_loaded_when_later_entry_fails_test() -> - ModuleA = gateway_hotpatch_loader_test_partial_a, - ModuleB = gateway_hotpatch_loader_test_partial_b, - cleanup_test_module(ModuleA), - cleanup_test_module(ModuleB), - try - BeamA1 = compile_test_module(ModuleA, 1), - BeamA2 = compile_test_module(ModuleA, 2), - BeamB1 = compile_test_module(ModuleB, 1), - BeamB2 = compile_test_module(ModuleB, 2), - {module, ModuleA} = load_test_beam(ModuleA, BeamA1), - {module, ModuleB} = load_test_beam(ModuleB, BeamB1), - {ok, Md5A1} = current_md5(ModuleA), - WrongExpectedMd5 = <<0:128>>, - {EntryA, _Md5A2} = beam_entry(ModuleA, Md5A1, BeamA2), - {EntryB, _Md5B2} = beam_entry(ModuleB, WrongExpectedMd5, BeamB2), - Bundle = #{ - version => 1, - build_sha => gateway_hotpatch_runtime:build_sha(), - modules => [EntryA, EntryB] - }, - ?assertMatch({error, {md5_mismatch, ModuleB, _, _}}, apply_bundle(Bundle)), - ?assertEqual(2, erlang:apply(ModuleA, version, [])), - ?assertEqual(1, erlang:apply(ModuleB, version, [])) - after - cleanup_test_module(ModuleA), - cleanup_test_module(ModuleB) - end. - -hex_test() -> - ?assertEqual(<<"0001020f10ff">>, hex(<<0, 1, 2, 15, 16, 255>>)). - -compile_test_module(Module, Version) -> - Dir = test_compile_dir(), - Source = filename:join(Dir, atom_to_list(Module) ++ ".erl"), - SourceText = io_lib:format( - "-module(~p).~n-export([version/0]).~nversion() -> ~p.~n", - [Module, Version] - ), - ok = file:write_file(Source, SourceText), - case compile:file(Source, [binary, return_errors, return_warnings]) of - {ok, Module, Beam} -> Beam; - {ok, Module, Beam, _Warnings} -> Beam; - Error -> erlang:error({test_module_compile_failed, Module, Error}) - end. - -test_compile_dir() -> - Root = - case os:getenv("TMPDIR") of - false -> "/tmp"; - Value -> Value - end, - Dir = filename:join(Root, "fluxer_hotpatch_loader_tests"), - ok = filelib:ensure_dir(filename:join(Dir, "placeholder")), - Dir. - -load_test_beam(Module, Beam) -> - cleanup_test_module(Module), - BeamPath = filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam"), - ok = file:write_file(BeamPath, Beam), - code:load_abs(filename:rootname(BeamPath)). - -beam_entry(Module, ExpectedMd5, TargetBeam) -> - {ok, TargetMd5} = beam_md5(TargetBeam), - {ok, BeamZstd} = compress_test_beam(TargetBeam), - { - #{ - module => Module, - expected_current_md5 => ExpectedMd5, - target_md5 => TargetMd5, - beam_zstd => BeamZstd - }, - TargetMd5 - }. - -compress_test_beam(Beam) -> - try erlang:apply(ezstd, compress, [Beam, 3]) of - Compressed when is_binary(Compressed) -> {ok, Compressed}; - {error, Reason} -> {error, Reason}; - Other -> {error, Other} - catch - Class:Reason -> {error, {Class, Reason}} - end. - -cleanup_test_module(Module) -> - _ = code:soft_purge(Module), - _ = code:purge(Module), - _ = code:delete(Module), - _ = code:purge(Module), - erase_hotpatch_loaded_md5(Module), - _ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".erl")), - _ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam")), - ok. - --spec erase_hotpatch_loaded_md5(atom()) -> ok. -erase_hotpatch_loaded_md5(Module) -> - _ = persistent_term:erase({?MODULE, loaded_md5, Module}), - ok. - --endif. diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl deleted file mode 100644 index 270ccd170..000000000 --- a/fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl +++ /dev/null @@ -1,319 +0,0 @@ -%% SPDX-License-Identifier: AGPL-3.0-or-later - --module(gateway_hotpatch_reconciler). --typing([eqwalizer]). - --behaviour(gen_server). - --export([start_link/0, is_ready/0, status/0, reconcile_async/0]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). - --record(state, { - enabled = false :: boolean(), - build_sha = <<"dev">> :: binary(), - public_keys = #{} :: #{binary() => binary()}, - applied_event_ids = [] :: [term()], - applied_count = 0 :: non_neg_integer(), - poll_interval_ms = 5000 :: pos_integer(), - last_error = undefined :: term() -}). - --type state() :: #state{}. - --define(SERVER, ?MODULE). --define(POLL, poll). --define(STARTUP_RECONCILE_RETRY_MS, 1000). - --spec start_link() -> gen_server:start_ret(). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - --spec is_ready() -> boolean(). -is_ready() -> - gateway_hotpatch_runtime:is_ready(). - --spec status() -> map(). -status() -> - gateway_hotpatch_runtime:status(). - --spec reconcile_async() -> ok. -reconcile_async() -> - gen_server:cast(?SERVER, reconcile). - --spec init([]) -> {ok, state()} | {stop, term()}. -init([]) -> - case gateway_hotpatch_runtime:is_enabled() of - false -> init_disabled(); - true -> init_enabled() - end. - --spec handle_call(term(), {pid(), term()}, state()) -> {reply, term(), state()}. -handle_call(status, _From, State) -> - {reply, state_status(State, gateway_hotpatch_runtime:is_ready()), State}; -handle_call(_Request, _From, State) -> - {reply, {error, unsupported_call}, State}. - --spec handle_cast(term(), state()) -> {noreply, state()}. -handle_cast(reconcile, State) -> - {noreply, reconcile_and_publish(State)}; -handle_cast(_Request, State) -> - {noreply, State}. - --spec handle_info(term(), state()) -> {noreply, state()}. -handle_info(?POLL, #state{enabled = true} = State) -> - NewState = reconcile_and_publish(State), - schedule_poll(NewState), - {noreply, NewState}; -handle_info(_Info, State) -> - {noreply, State}. - --spec terminate(term(), state()) -> ok. -terminate(_Reason, _State) -> - ok. - --spec code_change(term(), state(), term()) -> {ok, state()}. -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - --spec init_disabled() -> {ok, state()}. -init_disabled() -> - BuildSha = gateway_hotpatch_runtime:build_sha(), - State = #state{enabled = false, build_sha = BuildSha}, - gateway_hotpatch_runtime:put_ready(true), - publish_status(State, true), - {ok, State}. - --spec init_enabled() -> {ok, state()} | {stop, term()}. -init_enabled() -> - gateway_hotpatch_runtime:put_ready(false), - BuildSha = gateway_hotpatch_runtime:build_sha(), - PollIntervalMs = runtime_pos_integer(hotpatch_poll_interval_ms, 5000), - PublicKeyConfig = gateway_hotpatch_runtime:get(hotpatch_public_keys, undefined), - case gateway_hotpatch_bundle:parse_public_keys(PublicKeyConfig) of - {ok, PublicKeys} when map_size(PublicKeys) > 0 -> - State0 = #state{ - enabled = true, - build_sha = BuildSha, - public_keys = PublicKeys, - poll_interval_ms = max(1000, PollIntervalMs) - }, - publish_status(State0, false), - init_enabled_connected(State0); - {ok, _Empty} -> - init_enabled_config_error(BuildSha, PollIntervalMs, missing_hotpatch_public_keys); - {error, Reason} -> - init_enabled_config_error( - BuildSha, PollIntervalMs, {invalid_hotpatch_public_keys, Reason} - ) - end. - --spec init_enabled_config_error(binary(), pos_integer(), term()) -> {ok, state()}. -init_enabled_config_error(BuildSha, PollIntervalMs, Reason) -> - State = #state{ - enabled = true, - build_sha = BuildSha, - poll_interval_ms = max(1000, PollIntervalMs), - last_error = Reason - }, - gateway_hotpatch_runtime:put_ready(false), - publish_status(State, false), - logger:error("Gateway hotpatch configuration invalid: ~0tp", [Reason]), - schedule_poll(State), - {ok, State}. - --spec init_enabled_connected(state()) -> {ok, state()}. -init_enabled_connected(State0) -> - finish_enabled_startup(State0). - --spec finish_enabled_startup(state()) -> {ok, state()}. -finish_enabled_startup(State0) -> - TimeoutMs = runtime_pos_integer(hotpatch_startup_sync_timeout_ms, 30000), - DeadlineMs = erlang:monotonic_time(millisecond) + TimeoutMs, - finish_startup_reconcile(State0, startup_reconcile(State0, DeadlineMs)). - --spec finish_startup_reconcile(state(), {ok, state()} | {error, term()}) -> {ok, state()}. -finish_startup_reconcile(_State0, {ok, State}) -> - gateway_hotpatch_runtime:put_ready(true), - publish_status(State, true), - schedule_poll(State), - {ok, State}; -finish_startup_reconcile(State0, {error, Reason}) -> - Error = {hotpatch_startup_sync_failed, Reason}, - State = State0#state{last_error = Error}, - gateway_hotpatch_runtime:put_ready(false), - publish_status(State, false), - logger:error("Gateway hotpatch startup sync failed; keeping node unready: ~0tp", [Error]), - schedule_poll(State), - {ok, State}. - --spec startup_reconcile(state(), integer()) -> {ok, state()} | {error, term()}. -startup_reconcile(State, DeadlineMs) -> - case reconcile_once(State) of - {ok, NewState} -> - {ok, NewState}; - {error, Reason} -> - maybe_retry_startup_reconcile(State, DeadlineMs, Reason) - end. - --spec maybe_retry_startup_reconcile(state(), integer(), term()) -> - {ok, state()} | {error, term()}. -maybe_retry_startup_reconcile(State, DeadlineMs, Reason) -> - case erlang:monotonic_time(millisecond) >= DeadlineMs of - true -> - {error, Reason}; - false -> - retry_startup_reconcile(State#state{last_error = Reason}, DeadlineMs, Reason) - end. - --spec retry_startup_reconcile(state(), integer(), term()) -> {ok, state()} | {error, term()}. -retry_startup_reconcile(State, DeadlineMs, Reason) -> - case gateway_retry_timer:wait_until(?STARTUP_RECONCILE_RETRY_MS, DeadlineMs) of - ok -> startup_reconcile(State, DeadlineMs); - expired -> {error, Reason}; - {error, _InvalidDelay} -> {error, Reason} - end. - --spec reconcile_and_publish(state()) -> state(). -reconcile_and_publish(#state{enabled = false} = State) -> - gateway_hotpatch_runtime:put_ready(true), - publish_status(State, true), - State; -reconcile_and_publish(State) -> - case reconcile_once(State) of - {ok, NewState} -> - gateway_hotpatch_runtime:put_ready(true), - publish_status(NewState, true), - NewState; - {error, Reason} -> - publish_reconcile_error(State, Reason) - end. - --spec reconcile_once(state()) -> {ok, state()} | {error, term()}. -reconcile_once(#state{build_sha = BuildSha} = State) -> - case gateway_hotpatch_store:connect() of - ok -> fetch_and_apply_events(BuildSha, State); - {error, Reason} -> {error, {store_connect_failed, Reason}} - end. - --spec fetch_and_apply_events(binary(), state()) -> {ok, state()} | {error, term()}. -fetch_and_apply_events(BuildSha, State) -> - case gateway_hotpatch_store:fetch_events(BuildSha) of - {ok, Events} -> apply_events(Events, State); - {error, Reason} -> {error, {fetch_events_failed, Reason}} - end. - --spec publish_reconcile_error(state(), term()) -> state(). -publish_reconcile_error(State, {fetch_events_failed, _Reason} = Error) -> - ErrorState = State#state{last_error = Error}, - Ready = gateway_hotpatch_runtime:is_ready(), - gateway_hotpatch_runtime:put_ready(Ready), - publish_status(ErrorState, Ready), - logger:warning("Gateway hotpatch fetch failed; keeping current readiness: ~0tp", [Error]), - ErrorState; -publish_reconcile_error(State, Reason) -> - ErrorState = State#state{last_error = Reason}, - gateway_hotpatch_runtime:put_ready(false), - publish_status(ErrorState, false), - logger:error("Gateway hotpatch reconciliation failed: ~0tp", [Reason]), - ErrorState. - --spec apply_events([map()], state()) -> {ok, state()} | {error, term()}. -apply_events([], State) -> - {ok, State#state{last_error = undefined}}; -apply_events([Event | Rest], State) -> - EventId = maps:get(event_id, Event, undefined), - case lists:member(EventId, State#state.applied_event_ids) of - true -> - apply_events(Rest, State); - false -> - apply_new_event(EventId, Event, Rest, State) - end. - --spec apply_new_event(term(), map(), [map()], state()) -> {ok, state()} | {error, term()}. -apply_new_event(EventId, Event, Rest, State) -> - case apply_event(Event, State) of - {ok, NewState} -> apply_events(Rest, NewState); - {error, Reason} -> {error, {event_apply_failed, EventId, Reason}} - end. - --spec apply_event(map(), state()) -> {ok, state()} | {error, term()}. -apply_event(Event, #state{public_keys = PublicKeys, build_sha = BuildSha} = State) -> - EventId = maps:get(event_id, Event, undefined), - BundleHash = maps:get(bundle_sha256, Event, <<>>), - _ = code:ensure_loaded(gateway_hotpatch_loader), - case gateway_hotpatch_bundle:decode_signed_event(Event, PublicKeys) of - {ok, Bundle} -> - apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State); - {error, Reason} -> - audit_event(BuildSha, EventId, #{bundle_sha256 => BundleHash}, {error, Reason}), - {error, Reason} - end. - --spec apply_decoded_event(binary(), term(), binary(), map(), state()) -> - {ok, state()} | {error, term()}. -apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State) -> - Summary0 = #{bundle_sha256 => BundleHash}, - case gateway_hotpatch_loader:apply_bundle(Bundle) of - {ok, Summary} -> - handle_applied_event(BuildSha, EventId, Summary0, Summary, State); - {error, Reason} -> - audit_event(BuildSha, EventId, Summary0, {error, Reason}), - {error, Reason} - end. - --spec handle_applied_event(binary(), term(), map(), map(), state()) -> {ok, state()}. -handle_applied_event(BuildSha, EventId, Summary0, Summary, State) -> - AuditSummary = maps:merge(Summary0, Summary), - audit_event(BuildSha, EventId, AuditSummary, ok), - logger:notice("Applied gateway hotpatch event ~0tp summary=~0tp", [EventId, AuditSummary]), - {ok, State#state{ - applied_event_ids = [EventId | State#state.applied_event_ids], - applied_count = State#state.applied_count + 1, - last_error = undefined - }}. - --spec audit_event(binary(), term(), map(), ok | {error, term()}) -> ok. -audit_event(BuildSha, EventId, Summary, Result) when is_binary(EventId) -> - case - gateway_hotpatch_store:audit_applied( - BuildSha, gateway_hotpatch_runtime:node_name(), EventId, Summary, Result - ) - of - ok -> - ok; - {error, Reason} -> - logger:warning("Gateway hotpatch audit write failed: ~0tp", [Reason]), - ok - end; -audit_event(_BuildSha, _EventId, _Summary, _Result) -> - ok. - --spec publish_status(state(), boolean()) -> ok. -publish_status(State, Ready) -> - gateway_hotpatch_runtime:put_status(state_status(State, Ready)). - --spec state_status(state(), boolean()) -> map(). -state_status(State, Ready) -> - #{ - enabled => State#state.enabled, - ready => Ready, - build_sha => State#state.build_sha, - applied_count => State#state.applied_count, - applied_event_count => length(State#state.applied_event_ids), - last_error => State#state.last_error - }. - --spec schedule_poll(state()) -> ok. -schedule_poll(#state{poll_interval_ms = PollIntervalMs}) -> - erlang:send_after(PollIntervalMs, self(), ?POLL), - ok. - --spec runtime_pos_integer(atom(), pos_integer()) -> pos_integer(). -runtime_pos_integer(Key, Default) -> - normalize_pos_integer(gateway_hotpatch_runtime:get(Key, Default), Default). - --spec normalize_pos_integer(term(), pos_integer()) -> pos_integer(). -normalize_pos_integer(Value, _Default) when is_integer(Value), Value > 0 -> - Value; -normalize_pos_integer(_Value, Default) -> - Default. diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl deleted file mode 100644 index d878d7b17..000000000 --- a/fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl +++ /dev/null @@ -1,78 +0,0 @@ -%% SPDX-License-Identifier: AGPL-3.0-or-later - --module(gateway_hotpatch_runtime). --typing([eqwalizer]). - --export([ - build_sha/0, - node_name/0, - is_enabled/0, - get/2, - put_ready/1, - is_ready/0, - put_status/1, - status/0 -]). - --define(READY_KEY, {gateway_hotpatch, ready}). --define(STATUS_KEY, {gateway_hotpatch, status}). - --spec build_sha() -> binary(). -build_sha() -> - case os:getenv("BUILD_SHA") of - false -> build_version(); - "" -> build_version(); - Value -> type_conv:ensure_binary(Value, <<"dev">>) - end. - --spec build_version() -> binary(). -build_version() -> - case os:getenv("BUILD_VERSION") of - false -> <<"dev">>; - "" -> <<"dev">>; - Value -> type_conv:ensure_binary(Value, <<"dev">>) - end. - --spec node_name() -> binary(). -node_name() -> - atom_to_binary(node(), utf8). - --spec is_enabled() -> boolean(). -is_enabled() -> - case fluxer_gateway_env:get(hotpatch_enabled) of - true -> true; - _ -> false - end. - --spec get(atom(), term()) -> term(). -get(Key, Default) -> - case fluxer_gateway_env:get(Key) of - undefined -> Default; - Value -> Value - end. - --spec put_ready(boolean()) -> ok. -put_ready(Ready) when is_boolean(Ready) -> - persistent_term:put(?READY_KEY, Ready), - ok. - --spec is_ready() -> boolean(). -is_ready() -> - case persistent_term:get(?READY_KEY, true) of - false -> false; - _ -> true - end. - --spec put_status(map()) -> ok. -put_status(Status) -> - persistent_term:put(?STATUS_KEY, Status), - ok. - --spec status() -> map(). -status() -> - persistent_term:get(?STATUS_KEY, #{ - enabled => false, - ready => true, - build_sha => build_sha(), - applied_count => 0 - }). diff --git a/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl b/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl deleted file mode 100644 index 7759c70de..000000000 --- a/fluxer_gateway/src/gateway/gateway_hotpatch_store.erl +++ /dev/null @@ -1,277 +0,0 @@ -%% SPDX-License-Identifier: AGPL-3.0-or-later - --module(gateway_hotpatch_store). --typing([eqwalizer]). - --export([ - connect/0, - fetch_events/1, - append_event/5, - append_event/6, - audit_applied/5, - normalize_event_rows/1 -]). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. - --define(STMT_FETCH_EVENTS, gateway_hotpatch_fetch_events_by_build). --define(STMT_APPEND_EVENT, gateway_hotpatch_append_event). --define(STMT_AUDIT_APPLIED, gateway_hotpatch_audit_applied). - --spec connect() -> ok | {error, term()}. -connect() -> - case gateway_hotpatch_runtime:get(hotpatch_cassandra_hosts, undefined) of - Hosts when is_list(Hosts), Hosts =/= [] -> connect_hosts(Hosts); - Hosts when is_binary(Hosts), byte_size(Hosts) > 0 -> - start_erlcass(binary_to_list(Hosts)); - _ -> - {error, missing_hotpatch_cassandra_hosts} - end. - --spec connect_hosts(term()) -> ok | {error, term()}. -connect_hosts(Hosts) -> - case type_conv:to_list(Hosts) of - String when is_list(String), String =/= [] -> start_erlcass(String); - _ -> {error, missing_hotpatch_cassandra_hosts} - end. - --spec fetch_events(binary()) -> {ok, [map()]} | {error, term()}. -fetch_events(BuildSha) when is_binary(BuildSha) -> - case erlcass:execute(?STMT_FETCH_EVENTS, [BuildSha]) of - {ok, _Columns, Rows} -> {ok, normalize_event_rows(Rows)}; - ok -> {ok, []}; - {error, Reason} -> {error, Reason} - end. - --spec append_event(binary(), binary(), binary(), binary(), binary()) -> - {ok, binary()} | {error, term()}. -append_event(CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when - is_binary(CreatedBy), - is_binary(SignerKeyId), - is_binary(Signature), - is_binary(BundleSha256), - is_binary(Bundle) --> - BuildSha = gateway_hotpatch_runtime:build_sha(), - append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle). - --spec append_event(binary(), binary(), binary(), binary(), binary(), binary()) -> - {ok, binary()} | {error, term()}. -append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when - is_binary(BuildSha), - is_binary(CreatedBy), - is_binary(SignerKeyId), - is_binary(Signature), - is_binary(BundleSha256), - is_binary(Bundle) --> - case erlcass_uuid:gen_time() of - {ok, EventId} -> - Params = append_event_params( - BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle - ), - execute_append_event(EventId, Params); - {error, Reason} -> - {error, {event_id_failed, Reason}} - end. - --spec append_event_params( - binary(), binary(), binary(), binary(), binary(), binary(), binary() -) -> list(). -append_event_params(BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle) -> - [ - BuildSha, - EventId, - 1, - <<"beam_bundle">>, - CreatedBy, - SignerKeyId, - BundleSha256, - Signature, - Bundle - ]. - --spec execute_append_event(binary(), list()) -> {ok, binary()} | {error, term()}. -execute_append_event(EventId, Params) -> - case erlcass:execute(?STMT_APPEND_EVENT, Params) of - ok -> {ok, EventId}; - {ok, _Columns, _Rows} -> {ok, EventId}; - {error, Reason} -> {error, Reason} - end. - --spec audit_applied(binary(), binary(), binary(), map(), ok | {error, term()}) -> - ok | {error, term()}. -audit_applied(BuildSha, NodeName, EventId, Summary, Result) when - is_binary(BuildSha), is_binary(NodeName), is_binary(EventId), is_map(Summary) --> - ModuleCount = maps:get(module_count, Summary, 0), - BundleSha256 = maps:get(bundle_sha256, Summary, <<>>), - Status = audit_status(Result), - Error = audit_error(Result), - Params = [BuildSha, NodeName, EventId, ModuleCount, BundleSha256, Status, Error], - case erlcass:execute(?STMT_AUDIT_APPLIED, Params) of - ok -> ok; - {ok, _Columns, _Rows} -> ok; - {error, Reason} -> {error, Reason} - end. - --spec normalize_event_rows([list()]) -> [map()]. -normalize_event_rows(Rows) when is_list(Rows) -> - lists:filtermap(fun normalize_event_row/1, Rows). - --spec start_erlcass(string()) -> ok | {error, term()}. -start_erlcass(Hosts) -> - Keyspace = gateway_hotpatch_runtime:get(hotpatch_cassandra_keyspace, <<"fluxer">>), - Port = gateway_hotpatch_runtime:get(hotpatch_cassandra_port, 9042), - Options0 = [ - {contact_points, type_conv:ensure_binary(Hosts)}, - {port, Port}, - {latency_aware_routing, true}, - {token_aware_routing, true}, - {tcp_nodelay, true}, - {tcp_keepalive, {true, 60}}, - {connect_timeout, 5000}, - {request_timeout, 5000}, - {retry_policy, {default, true}}, - {default_consistency_level, 6} - ], - case maybe_credentials(Options0) of - {ok, Options} -> start_erlcass_with_options(Keyspace, Options); - {error, Reason} -> {error, Reason} - end. - --spec start_erlcass_with_options(binary(), list()) -> ok | {error, term()}. -start_erlcass_with_options(Keyspace, Options) -> - application:set_env(erlcass, keyspace, Keyspace), - application:set_env(erlcass, cluster_options, Options), - application:set_env(erlcass, log_level, 2), - case application:ensure_all_started(erlcass) of - {ok, _Apps} -> prepare_statements(); - {error, {erlcass, {already_started, erlcass}}} -> prepare_statements(); - {error, Reason} -> {error, {erlcass_start_failed, Reason}} - end. - --spec prepare_statements() -> ok | {error, term()}. -prepare_statements() -> - Statements = [ - {?STMT_FETCH_EVENTS, << - "SELECT event_id, schema_version, kind, created_by, signer_key_id, " - "bundle_sha256, signature, bundle " - "FROM gateway_hotpatch_events_by_build WHERE build_sha = ?" - >>}, - {?STMT_APPEND_EVENT, << - "INSERT INTO gateway_hotpatch_events_by_build " - "(build_sha, event_id, schema_version, kind, created_at, created_by, " - "signer_key_id, bundle_sha256, signature, bundle) " - "VALUES (?, ?, ?, ?, toTimestamp(now()), ?, ?, ?, ?, ?)" - >>}, - {?STMT_AUDIT_APPLIED, << - "INSERT INTO gateway_hotpatch_applied_by_node " - "(build_sha, node_name, event_id, applied_at, module_count, bundle_sha256, " - "status, error) " - "VALUES (?, ?, ?, toTimestamp(now()), ?, ?, ?, ?)" - >>} - ], - prepare_statements(Statements). - --spec prepare_statements([{atom(), binary()}]) -> ok | {error, term()}. -prepare_statements([]) -> - ok; -prepare_statements([{Id, Query} | Rest]) -> - case prepare_statement(Id, Query) of - ok -> prepare_statements(Rest); - {error, Reason} -> {error, Reason} - end. - --spec prepare_statement(atom(), binary()) -> ok | {error, term()}. -prepare_statement(Id, Query) -> - case erlcass:add_prepare_statement(Id, Query) of - ok -> ok; - {error, already_exist} -> ok; - {error, Reason} -> {error, {prepare_failed, Id, Reason}} - end. - --spec maybe_credentials(list()) -> {ok, list()} | {error, term()}. -maybe_credentials(Options) -> - Username = gateway_hotpatch_runtime:get(hotpatch_cassandra_username, undefined), - Password = gateway_hotpatch_runtime:get(hotpatch_cassandra_password, undefined), - case {Username, Password} of - {U, P} when is_binary(U), byte_size(U) > 0, is_binary(P), byte_size(P) > 0 -> - {ok, [{credentials, {U, P}} | Options]}; - _ -> - {error, missing_hotpatch_cassandra_credentials} - end. - --spec normalize_event_row(list()) -> {true, map()} | false. -normalize_event_row([ - EventId, - SchemaVersion, - Kind, - CreatedBy, - SignerKeyId, - BundleSha256, - Signature, - Bundle -]) when - is_binary(EventId), - is_binary(SignerKeyId), - is_binary(BundleSha256), - is_binary(Signature), - is_binary(Bundle) --> - {true, #{ - event_id => EventId, - schema_version => SchemaVersion, - kind => Kind, - created_by => CreatedBy, - signer_key_id => SignerKeyId, - bundle_sha256 => BundleSha256, - signature => Signature, - bundle => Bundle - }}; -normalize_event_row(_Row) -> - false. - --spec audit_status(ok | {error, term()}) -> binary(). -audit_status(ok) -> <<"ok">>; -audit_status({error, _Reason}) -> <<"error">>. - --spec audit_error(ok | {error, term()}) -> binary(). -audit_error(ok) -> - <<>>; -audit_error({error, Reason}) -> - iolist_to_binary(io_lib:format("~0tp", [Reason])). - --ifdef(TEST). - -normalize_event_rows_test() -> - EventId = <<1:128>>, - Row = [ - EventId, - 1, - <<"beam_bundle">>, - <<"hampus">>, - <<"ops">>, - <<2:256>>, - <<3:512>>, - <<"bundle">> - ], - ?assertEqual( - [ - #{ - event_id => EventId, - schema_version => 1, - kind => <<"beam_bundle">>, - created_by => <<"hampus">>, - signer_key_id => <<"ops">>, - bundle_sha256 => <<2:256>>, - signature => <<3:512>>, - bundle => <<"bundle">> - } - ], - normalize_event_rows([Row, [invalid]]) - ). - --endif. diff --git a/fluxer_gateway/src/gateway/gateway_node_router.erl b/fluxer_gateway/src/gateway/gateway_node_router.erl index a7c2cf1f5..b2374c9f3 100644 --- a/fluxer_gateway/src/gateway/gateway_node_router.erl +++ b/fluxer_gateway/src/gateway/gateway_node_router.erl @@ -98,7 +98,7 @@ fallback_owner_for_role(Role) -> -spec is_ready() -> boolean(). is_ready() -> - not is_draining() andalso gateway_hotpatch_reconciler:is_ready(). + not is_draining(). -spec is_draining() -> boolean(). is_draining() -> diff --git a/fluxer_gateway/src/gateway/metrics_handler.erl b/fluxer_gateway/src/gateway/metrics_handler.erl index 2c5f3bb4e..12300c1c5 100644 --- a/fluxer_gateway/src/gateway/metrics_handler.erl +++ b/fluxer_gateway/src/gateway/metrics_handler.erl @@ -23,7 +23,6 @@ render_metrics() -> render_cluster_counters(), render_process_counts(), render_push_dispatcher_stats(), - render_hotpatch_metrics(), render_vm_metrics() ]. @@ -256,33 +255,6 @@ render_push_dispatcher_stats() -> ] end. --spec render_hotpatch_metrics() -> iolist(). -render_hotpatch_metrics() -> - Status = safe_apply_map(fun gateway_hotpatch_reconciler:status/0), - Ready = maps:get(ready, Status, true), - Enabled = maps:get(enabled, Status, false), - AppliedCount = maps:get(applied_count, Status, 0), - [ - format_metric( - <<"fluxer_gateway_hotpatch_enabled">>, - <<"gauge">>, - <<"Gateway hotpatch reconciler enabled">>, - bool_metric(Enabled) - ), - format_metric( - <<"fluxer_gateway_hotpatch_ready">>, - <<"gauge">>, - <<"Gateway hotpatch reconciliation readiness">>, - bool_metric(Ready) - ), - format_metric( - <<"fluxer_gateway_hotpatch_applied_events_total">>, - <<"counter">>, - <<"Gateway hotpatch events applied on this node">>, - integer_to_binary(AppliedCount) - ) - ]. - -spec render_vm_metrics() -> iolist(). render_vm_metrics() -> Memory = safe_apply_list(fun erlang:memory/0), @@ -376,7 +348,3 @@ format_labeled_series(Name, Type, Help, LabelValues) -> || {Label, Value} <- LabelValues ] ]. - --spec bool_metric(term()) -> binary(). -bool_metric(true) -> <<"1">>; -bool_metric(_) -> <<"0">>. diff --git a/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl b/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl index 1b9c46615..ee09298c9 100644 --- a/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl +++ b/fluxer_gateway/src/guild/guild_member_list_sync_batch.erl @@ -159,7 +159,7 @@ flush_pending_syncs_clears_empty_batch_test() -> State = #{?SYNC_BATCH_STATE_KEY => #{pending_list_ids => #{}}}, ?assertEqual(#{}, flush_pending_syncs(State)). -queue_list_sync_drains_pre_hotpatch_pending_batch_test() -> +queue_list_sync_drains_preexisting_pending_batch_test() -> Ref = erlang:send_after(60000, self(), ?FLUSH_SYNC_BATCH_MSG), State = #{ ?SYNC_BATCH_STATE_KEY => #{ diff --git a/fluxer_svc/src/postgres.rs b/fluxer_svc/src/postgres.rs index 391f7410c..f4ea1dd7c 100644 --- a/fluxer_svc/src/postgres.rs +++ b/fluxer_svc/src/postgres.rs @@ -11,6 +11,9 @@ use std::str::FromStr; use tokio_postgres::{Config as PgConfig, Row, config::SslMode, types::ToSql}; use tokio_postgres_rustls::MakeRustlsConnect; +const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE: i32 = 0x4658_4b56; +const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT: &str = "120s"; + #[derive(Clone, Debug)] pub struct PostgresConfig { pub url: Option, @@ -134,8 +137,30 @@ pub async fn ensure_kv_schema(pool: &Pool, kv_table: &str) -> anyhow::Result<()> let messages_message_index = quote_identifier(&format!("{kv_table}_messages_message_idx"))?; let message_reactions_message_index = quote_identifier(&format!("{kv_table}_message_reactions_message_idx"))?; - let client = pool.get().await?; - client + let mut client = pool.get().await?; + let transaction = client + .transaction() + .await + .context("failed to begin Postgres KV schema transaction")?; + transaction + .query_one( + "SELECT set_config('statement_timeout', $1, true)", + &[&POSTGRES_KV_SCHEMA_LOCK_TIMEOUT], + ) + .await + .context("failed to configure Postgres KV schema lock timeout")?; + transaction + .query_one( + "SELECT pg_advisory_xact_lock($1, hashtext($2))", + &[&POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, &kv_table], + ) + .await + .context("failed to acquire Postgres KV schema lock")?; + transaction + .query_one("SELECT set_config('statement_timeout', '0', true)", &[]) + .await + .context("failed to clear Postgres KV schema lock timeout")?; + transaction .batch_execute(&format!( r#" CREATE TABLE IF NOT EXISTS {table} ( @@ -162,6 +187,10 @@ DROP INDEX IF EXISTS {old_partition_index}; )) .await .context("failed to ensure Postgres KV schema")?; + transaction + .commit() + .await + .context("failed to commit Postgres KV schema transaction")?; Ok(()) } diff --git a/package.json b/package.json index 1622ff3a5..7e282a45f 100644 --- a/package.json +++ b/package.json @@ -13,11 +13,10 @@ "dev:desktop:canary": "cargo run -p fluxer-dev -- desktop canary", "dev:desktop:package": "cargo run -p fluxer-dev -- desktop package", "dev:desktop:run": "cargo run -p fluxer-dev -- desktop run", - "dev:infra:down": "docker compose -f .devcontainer/docker-compose.yml down", - "dev:infra:restart": "pnpm dev:infra:down && pnpm dev:infra:up", - "dev:infra:up": "pnpm dev:infra:up:lite", - "dev:infra:up:full": "docker compose -f .devcontainer/docker-compose.yml up -d postgres cassandra valkey nats livekit elasticsearch meilisearch mailpit", - "dev:infra:up:lite": "docker compose -f .devcontainer/docker-compose.yml up -d postgres valkey nats livekit meilisearch mailpit", + "dev:infra:restart": "pnpm dev:infra:stop && pnpm dev:infra:start", + "dev:infra:start": "cargo run -p fluxer-dev -- infra start", + "dev:infra:status": "cargo run -p fluxer-dev -- infra status", + "dev:infra:stop": "cargo run -p fluxer-dev -- infra stop", "dev:services": "cargo run -p fluxer-dev -- rust-services", "dev:tunnel": "cargo run -p fluxer-dev -- dev --cloudflare-tunnel", "dev:tunnel:configure": "cargo run -p fluxer-dev -- tunnel configure", diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.ts index 6c7e5df55..e2165d47f 100644 --- a/packages/config/src/ConfigLoader.ts +++ b/packages/config/src/ConfigLoader.ts @@ -195,6 +195,7 @@ function defaultConfig(): MasterConfig { api_key: '', api_secret: '', url: '', + internal_url: '', webhook_url: '', }, search: { diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.ts index 28c719942..629e55c19 100644 --- a/packages/config/src/MasterConfig.ts +++ b/packages/config/src/MasterConfig.ts @@ -243,6 +243,7 @@ export interface MasterConfig { api_key: string; api_secret: string; url: string; + internal_url: string; webhook_url: string; default_region?: { id: string; diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts index adb9175ce..dae1be5af 100644 --- a/packages/config/src/config_loader/EnvironmentOverrides.ts +++ b/packages/config/src/config_loader/EnvironmentOverrides.ts @@ -266,6 +266,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_LIVEKIT_API_KEY: {path: ['integrations', 'voice', 'api_key']}, FLUXER_LIVEKIT_API_SECRET: {path: ['integrations', 'voice', 'api_secret']}, FLUXER_LIVEKIT_URL: {path: ['integrations', 'voice', 'url']}, + FLUXER_LIVEKIT_INTERNAL_URL: {path: ['integrations', 'voice', 'internal_url']}, FLUXER_LIVEKIT_WEBHOOK_URL: {path: ['integrations', 'voice', 'webhook_url']}, FLUXER_LIVEKIT_DEFAULT_REGION: {path: ['integrations', 'voice', 'default_region'], parse: parseEnvValue}, FLUXER_SEARCH_ENGINE: {path: ['integrations', 'search', 'engine']}, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83c0a9173..97cb49d5f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7701,8 +7701,8 @@ packages: resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==} engines: {node: '>=16'} - caniuse-lite@1.0.30001774: - resolution: {integrity: sha512-DDdwPGz99nmIEv216hKSgLD+D4ikHQHjBC/seF98N9CPqRX4M5mSxT9eTV6oyisnJcuzxtZy4n17yKKQYmYQOA==} + caniuse-lite@1.0.30001810: + resolution: {integrity: sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==} cassandra-driver@4.8.0: resolution: {integrity: sha512-HritfMGq9V7SuESeSodHvArs0mLuMk7uh+7hQK2lqdvXrvm50aWxb4RPxkK3mPDdsgHjJ427xNRFITMH2ei+Sw==} @@ -17551,7 +17551,7 @@ snapshots: autoprefixer@10.4.24(postcss@8.5.6): dependencies: browserslist: 4.28.1 - caniuse-lite: 1.0.30001774 + caniuse-lite: 1.0.30001810 fraction.js: 5.3.4 picocolors: 1.1.1 postcss: 8.5.6 @@ -17666,7 +17666,7 @@ snapshots: browserslist@4.28.1: dependencies: baseline-browser-mapping: 2.10.0 - caniuse-lite: 1.0.30001774 + caniuse-lite: 1.0.30001810 electron-to-chromium: 1.5.302 node-releases: 2.0.27 update-browserslist-db: 1.2.3(browserslist@4.28.1) @@ -17761,7 +17761,7 @@ snapshots: camelcase@8.0.0: {} - caniuse-lite@1.0.30001774: {} + caniuse-lite@1.0.30001810: {} cassandra-driver@4.8.0: dependencies: diff --git a/tools/ci/src/app_dev_server.rs b/tools/ci/src/app_dev_server.rs index c63868325..cbf5d92c5 100644 --- a/tools/ci/src/app_dev_server.rs +++ b/tools/ci/src/app_dev_server.rs @@ -4,9 +4,11 @@ use crate::app_wasm::resolve_app_dir; use anyhow::{Context, Result, bail, ensure}; use clap::Args; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::ffi::OsStr; use std::fs; +use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -23,11 +25,51 @@ pub struct AppDevServerArgs { app_dir: Option, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(untagged)] +enum InputFingerprint { + Sha256 { + sha256: String, + #[serde(skip)] + legacy_timestamp_ms: Option, + }, + LegacyTimestamp(f64), +} + +impl InputFingerprint { + fn matches(&self, current: &Self) -> bool { + match (self, current) { + ( + Self::Sha256 { sha256, .. }, + Self::Sha256 { + sha256: current, .. + }, + ) => sha256 == current, + ( + Self::LegacyTimestamp(timestamp), + Self::Sha256 { + legacy_timestamp_ms: Some(current), + .. + }, + ) => timestamp == current, + (Self::LegacyTimestamp(timestamp), Self::LegacyTimestamp(current)) => { + timestamp == current + } + _ => false, + } + } + + fn is_legacy(&self) -> bool { + matches!(self, Self::LegacyTimestamp(_)) + } +} + +type StepInputs = BTreeMap; + #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] struct StepMetadata { - last_run: f64, - inputs: BTreeMap, + inputs: StepInputs, } type Metadata = BTreeMap; @@ -103,20 +145,14 @@ impl AppDevServer { ) .await?; - if env_truthy("FLUXER_APP_SKIP_I18N_COMPILE") { - eprintln!("Skipping pnpm lingui:compile because FLUXER_APP_SKIP_I18N_COMPILE is set."); - } else { - self.run_cached_step( - "lingui", - gather_lingui_inputs, - "pnpm lingui:compile", - |server, shutdown| { - Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown)) - }, - &mut shutdown_rx, - ) - .await?; - } + self.run_cached_step( + "lingui", + gather_lingui_inputs, + "pnpm lingui:compile", + |server, shutdown| Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown)), + &mut shutdown_rx, + ) + .await?; if *shutdown_rx.borrow() { return Ok(()); @@ -175,35 +211,50 @@ impl AppDevServer { shutdown: &mut watch::Receiver, ) -> Result<()> where - G: Fn(&Path) -> Result>, + G: Fn(&Path) -> Result, E: for<'a> FnOnce( &'a AppDevServer, &'a mut watch::Receiver, ) -> std::pin::Pin> + 'a>>, { + if *shutdown.borrow() { + return Ok(()); + } let inputs = gather_inputs(&self.project_root)?; if !self.should_run_step(step_name, &inputs) { + let needs_upgrade = self + .metadata + .get(step_name) + .is_some_and(|entry| entry.inputs.values().any(InputFingerprint::is_legacy)); + if needs_upgrade { + self.metadata + .insert(step_name.to_string(), StepMetadata { inputs }); + self.save_metadata()?; + } println!("Skipping {label} (no changes detected)"); return Ok(()); } execute(self, shutdown).await?; - self.metadata.insert( - step_name.to_string(), - StepMetadata { - last_run: timestamp_ms(SystemTime::now())?, - inputs, - }, - ); + if *shutdown.borrow() { + return Ok(()); + } + self.metadata + .insert(step_name.to_string(), StepMetadata { inputs }); self.save_metadata() } - fn should_run_step(&self, step_name: &str, inputs: &BTreeMap) -> bool { + fn should_run_step(&self, step_name: &str, inputs: &StepInputs) -> bool { let Some(entry) = self.metadata.get(step_name) else { return true; }; - &entry.inputs != inputs + entry.inputs.len() != inputs.len() + || entry.inputs.iter().any(|(path, cached)| { + inputs + .get(path) + .is_none_or(|current| !cached.matches(current)) + }) } async fn run_command( @@ -286,7 +337,7 @@ impl AppDevServer { } } -fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result> { +fn collect_file_digests(project_root: &Path, paths: &[PathBuf]) -> Result { let mut result = BTreeMap::new(); for rel_path in paths { let absolute_path = project_root.join(rel_path); @@ -297,16 +348,16 @@ fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result( +fn collect_directory_digests

( project_root: &Path, root_rel: &Path, predicate: P, -) -> Result> +) -> Result where P: Fn(&str) -> bool, { @@ -335,20 +386,47 @@ where if !predicate(&key) { continue; } - result.insert(key, timestamp_ms(entry.metadata()?.modified()?)?); + result.insert(key, fingerprint_file(entry.path())?); } Ok(result) } +fn hash_file(path: &Path) -> Result { + let mut file = fs::File::open(path) + .with_context(|| format!("Failed to open {} for hashing", path.display()))?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let bytes_read = file + .read(&mut buffer) + .with_context(|| format!("Failed to read {} for hashing", path.display()))?; + if bytes_read == 0 { + break; + } + hasher.update(&buffer[..bytes_read]); + } + Ok(hex::encode(hasher.finalize())) +} + +fn fingerprint_file(path: &Path) -> Result { + let sha256 = hash_file(path)?; + let metadata = fs::metadata(path) + .with_context(|| format!("Failed to stat {} after hashing", path.display()))?; + Ok(InputFingerprint::Sha256 { + sha256, + legacy_timestamp_ms: Some(timestamp_ms(metadata.modified()?)?), + }) +} + fn should_walk_entry(path: &Path, skip_dirs: &BTreeSet<&str>) -> bool { path.file_name() .and_then(OsStr::to_str) .is_none_or(|name| !skip_dirs.contains(name)) } -fn gather_wasm_inputs(project_root: &Path) -> Result> { +fn gather_wasm_inputs(project_root: &Path) -> Result { let markdown_parser_rust_dir = PathBuf::from("../packages/markdown_parser/rust"); - let mut inputs = collect_file_stats( + let mut inputs = collect_file_digests( project_root, &[ PathBuf::from("../tools/ci/Cargo.toml"), @@ -361,12 +439,12 @@ fn gather_wasm_inputs(project_root: &Path) -> Result> { markdown_parser_rust_dir.join("Cargo.toml"), ], )?; - inputs.extend(collect_directory_stats( + inputs.extend(collect_directory_digests( project_root, Path::new("rust/libfluxcore"), |path| !path.contains("/target/"), )?); - inputs.extend(collect_directory_stats( + inputs.extend(collect_directory_digests( project_root, &markdown_parser_rust_dir, |path| !path.contains("/target/"), @@ -374,15 +452,15 @@ fn gather_wasm_inputs(project_root: &Path) -> Result> { Ok(inputs) } -fn gather_color_inputs(project_root: &Path) -> Result> { - collect_file_stats( +fn gather_color_inputs(project_root: &Path) -> Result { + collect_file_digests( project_root, &[PathBuf::from("scripts/GenerateColorSystem.ts")], ) } -fn gather_message_layout_inputs(project_root: &Path) -> Result> { - collect_file_stats( +fn gather_message_layout_inputs(project_root: &Path) -> Result { + collect_file_digests( project_root, &[ PathBuf::from("scripts/GenerateMessageLayoutCss.ts"), @@ -391,8 +469,8 @@ fn gather_message_layout_inputs(project_root: &Path) -> Result Result> { - collect_file_stats( +fn gather_mask_inputs(project_root: &Path) -> Result { + collect_file_digests( project_root, &[ PathBuf::from("scripts/GenerateAvatarMasks.ts"), @@ -401,14 +479,14 @@ fn gather_mask_inputs(project_root: &Path) -> Result> { ) } -fn gather_css_module_inputs(project_root: &Path) -> Result> { - collect_directory_stats(project_root, Path::new("src"), |path| { +fn gather_css_module_inputs(project_root: &Path) -> Result { + collect_directory_digests(project_root, Path::new("src"), |path| { path.ends_with(".module.css") }) } -fn gather_lingui_inputs(project_root: &Path) -> Result> { - collect_directory_stats( +fn gather_lingui_inputs(project_root: &Path) -> Result { + collect_directory_digests( project_root, Path::new("src/features/i18n/locales"), |path| path.ends_with(".po"), @@ -493,12 +571,6 @@ fn timestamp_ms(timestamp: SystemTime) -> Result { * 1000.0) } -fn env_truthy(name: &str) -> bool { - std::env::var(name) - .ok() - .is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true")) -} - fn display_command(command: &str, args: &[&str]) -> String { std::iter::once(command.to_string()) .chain(args.iter().map(|arg| quote_arg(arg))) diff --git a/tools/dev/cassandra_target_schema.json b/tools/dev/cassandra_target_schema.json index 7aea67521..797962a5e 100644 --- a/tools/dev/cassandra_target_schema.json +++ b/tools/dev/cassandra_target_schema.json @@ -9194,92 +9194,6 @@ ], "primary_key": "((guild_id), webhook_id)", "options": "" - }, - { - "name": "gateway_hotpatch_events_by_build", - "columns": [ - { - "name": "build_sha", - "type": "text" - }, - { - "name": "event_id", - "type": "timeuuid" - }, - { - "name": "schema_version", - "type": "int" - }, - { - "name": "kind", - "type": "text" - }, - { - "name": "created_at", - "type": "timestamp" - }, - { - "name": "created_by", - "type": "text" - }, - { - "name": "signer_key_id", - "type": "text" - }, - { - "name": "bundle_sha256", - "type": "blob" - }, - { - "name": "signature", - "type": "blob" - }, - { - "name": "bundle", - "type": "blob" - } - ], - "primary_key": "((build_sha), event_id)", - "options": "CLUSTERING ORDER BY (event_id ASC)" - }, - { - "name": "gateway_hotpatch_applied_by_node", - "columns": [ - { - "name": "build_sha", - "type": "text" - }, - { - "name": "node_name", - "type": "text" - }, - { - "name": "event_id", - "type": "timeuuid" - }, - { - "name": "applied_at", - "type": "timestamp" - }, - { - "name": "module_count", - "type": "int" - }, - { - "name": "bundle_sha256", - "type": "blob" - }, - { - "name": "status", - "type": "text" - }, - { - "name": "error", - "type": "text" - } - ], - "primary_key": "((build_sha, node_name), event_id)", - "options": "CLUSTERING ORDER BY (event_id ASC)" } ], "indexes": [ diff --git a/tools/dev/src/bootstrap.rs b/tools/dev/src/bootstrap.rs index 41879683c..74a4232d9 100644 --- a/tools/dev/src/bootstrap.rs +++ b/tools/dev/src/bootstrap.rs @@ -1,18 +1,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use crate::desktop::install_desktop; use crate::gateway::setup_gateway_config; -use crate::object_store::{bootstrap_schema_and_object_store, wait_s3_api}; +use crate::object_store::bootstrap_schema; use crate::paths::{ensure_state_dirs, ensure_writable_dev_paths}; -use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http, wait_tcp}; +use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http_success, wait_tcp}; use anyhow::Result; -pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result<()> { +pub async fn bootstrap(skip_install: bool) -> Result<()> { ensure_state_dirs()?; ensure_writable_dev_paths()?; if !skip_install { - crate::proc::run(&["corepack", "enable"])?; - crate::proc::run(&["corepack", "prepare", "pnpm@10.29.3", "--activate"])?; run_command( &["pnpm", "install", "--frozen-lockfile"], RunOptions { @@ -23,13 +20,10 @@ pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result ..RunOptions::default() }, )?; - if !skip_desktop_install { - install_desktop()?; - } } setup_gateway_config()?; wait_core_infra().await?; - bootstrap_schema_and_object_store().await?; + bootstrap_schema().await?; println!("Fluxer dev bootstrap complete."); Ok(()) } @@ -43,16 +37,11 @@ pub async fn post_start() -> Result<()> { } pub async fn wait_core_infra() -> Result<()> { + wait_tcp("Postgres", "postgres", 5432, 120).await?; + wait_http_success("Meilisearch", "http://meilisearch:7700/health", 120).await?; wait_tcp("Valkey", "valkey", 6379, 120).await?; wait_tcp("NATS", "nats", 4222, 120).await?; wait_tcp("LiveKit", "livekit", 7880, 120).await?; crate::media_proxy::ensure_dev_object_store(true, 120).await?; - wait_tcp("SeaweedFS S3", "127.0.0.1", 8333, 120).await?; - wait_http( - "SeaweedFS master", - "http://127.0.0.1:9333/cluster/status", - 120, - ) - .await?; - wait_s3_api(120).await + Ok(()) } diff --git a/tools/dev/src/dev.rs b/tools/dev/src/dev.rs index 6d5769e88..1dc8c832c 100644 --- a/tools/dev/src/dev.rs +++ b/tools/dev/src/dev.rs @@ -1,15 +1,16 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use crate::gateway::setup_gateway_config; +use crate::gateway::{build_gateway_cluster_nodes, setup_gateway_config}; use crate::manifest::{ - ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_PORT, LOOPBACK_HOST, - MEDIA_PROXY_PORT, rust_services, + ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_GATEWAY_PORTS_ENV, + DEV_PROXY_PORT, GATEWAY_PORT, LOOPBACK_HOST, MEDIA_PROXY_PORT, rust_services, }; use crate::object_store::s3_endpoint; use crate::paths::{DESKTOP_DIR, ROOT}; use crate::proc::{ - PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, - merged_env, restart_budget_exceeded, run_command, wait_http, + AwaitOutcome, PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, + await_or_shutdown, configure_process_group, format_command, merged_env, prefix_output, + restart_budget_exceeded, run_command_interruptible, wait_http, }; use anyhow::{Context, Result, bail}; use std::collections::{BTreeMap, VecDeque}; @@ -30,10 +31,10 @@ const DEFAULT_TASKS: &[&str] = &[ "proxy", "services", "media", - "gateway", "marketing", "admin", "api", + "gateway-single", "worker", "app", "app-proxy", @@ -63,7 +64,7 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result Result {} + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev startup..."); + return Ok(0); + } + } + match await_or_shutdown(&mut shutdown, ensure_object_store_if_needed(&selected)).await { + AwaitOutcome::Completed(result) => result?, + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev startup..."); + return Ok(0); + } + } + match await_or_shutdown(&mut shutdown, wait_for_search_backend_if_needed(&selected)).await { + AwaitOutcome::Completed(result) => result?, + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev startup..."); + return Ok(0); + } + } + setup_gateway_config()?; let mut processes = Vec::new(); let mut task_names = Vec::new(); let mut task_specs = Vec::new(); @@ -102,29 +122,82 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result {} + AwaitOutcome::Completed(Err(error)) => { + crate::gateway::stop_processes(&mut processes); + return Err(error); + } + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev tasks..."); + crate::gateway::stop_processes(&mut processes); + return Ok(0); + } + } } - let process = start_task(tasks.get(name.as_str()).expect("validated task"))?; + let process = match start_task(tasks.get(name.as_str()).expect("validated task")) { + Ok(process) => process, + Err(error) => { + crate::gateway::stop_processes(&mut processes); + return Err(error); + } + }; processes.push(process); task_names.push(name.clone()); task_specs.push(tasks.get(name.as_str()).expect("validated task").clone()); task_restarts.push(VecDeque::new()); - if name == "services" - && let Err(error) = wait_for_rust_services(&mut processes).await - { - crate::gateway::stop_processes(&mut processes); - return Err(error); + if name == "services" { + match await_or_shutdown(&mut shutdown, wait_for_rust_services(&mut processes)).await { + AwaitOutcome::Completed(Ok(())) => {} + AwaitOutcome::Completed(Err(error)) => { + crate::gateway::stop_processes(&mut processes); + return Err(error); + } + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev tasks..."); + crate::gateway::stop_processes(&mut processes); + return Ok(0); + } + } } - if name == "api" - && let Err(error) = wait_for_api(&mut processes).await - { - crate::gateway::stop_processes(&mut processes); - return Err(error); + if name == "gateway" || name == "gateway-single" { + match await_or_shutdown( + &mut shutdown, + wait_for_gateway(&mut processes, gateway_readiness_port), + ) + .await + { + AwaitOutcome::Completed(Ok(())) => {} + AwaitOutcome::Completed(Err(error)) => { + crate::gateway::stop_processes(&mut processes); + return Err(error); + } + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev tasks..."); + crate::gateway::stop_processes(&mut processes); + return Ok(0); + } + } + } + if name == "api" { + match await_or_shutdown(&mut shutdown, wait_for_api(&mut processes)).await { + AwaitOutcome::Completed(Ok(())) => {} + AwaitOutcome::Completed(Err(error)) => { + crate::gateway::stop_processes(&mut processes); + return Err(error); + } + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev tasks..."); + crate::gateway::stop_processes(&mut processes); + return Ok(0); + } + } } } loop { @@ -135,9 +208,17 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result= next_object_store_check { - if let Err(error) = monitor_object_store_dependency().await { - crate::gateway::stop_processes(&mut processes); - return Err(error); + match await_or_shutdown(&mut shutdown, monitor_object_store_dependency()).await { + AwaitOutcome::Completed(Ok(())) => {} + AwaitOutcome::Completed(Err(error)) => { + crate::gateway::stop_processes(&mut processes); + return Err(error); + } + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping dev tasks..."); + crate::gateway::stop_processes(&mut processes); + return Ok(0); + } } next_object_store_check = Instant::now() + object_store_monitor_interval; } @@ -152,6 +233,38 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result, + selected: &[String], +) -> Result { + let cluster_selected = selected.iter().any(|name| name == "gateway"); + let single_selected = selected.iter().any(|name| name == "gateway-single"); + if cluster_selected && single_selected { + bail!("Select either `gateway` or `gateway-single`, not both"); + } + let ports = if cluster_selected { + build_gateway_cluster_nodes()? + .into_iter() + .filter(|node| node.role == "websocket") + .map(|node| node.http_port) + .collect::>() + } else { + vec![GATEWAY_PORT] + }; + let value = ports + .iter() + .map(u16::to_string) + .collect::>() + .join(","); + let proxy = tasks + .get_mut("proxy") + .context("dev task table is missing the proxy task")?; + proxy + .env + .push((DEV_PROXY_GATEWAY_PORTS_ENV.to_owned(), Some(value))); + Ok(*ports.first().expect("gateway proxy ports are non-empty")) +} + async fn ensure_object_store_if_needed(selected: &[String]) -> Result<()> { if !selected_needs_object_store(selected) { return Ok(()); @@ -202,6 +315,7 @@ fn restart_exited_tasks( let Some(status) = process.try_wait()? else { continue; }; + crate::gateway::stop_child_processes(&mut [process]); if restart_budget_exceeded(&mut task_restarts[index], Instant::now()) { bail!( "Dev task {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up", @@ -237,6 +351,10 @@ fn task_table_for_availability( marketing_availability: MarketingAvailability, ) -> Result> { let self_tool = self_tool_command()?; + let api_dir = ROOT.join("fluxer_api"); + let api_tsx = api_dir.join("node_modules/.bin/tsx"); + let app_dir = ROOT.join("fluxer_app"); + let app_dev_server = ROOT.join("tools/ci/run.sh"); let public_url = public_url(); let marketing_endpoint = std::env::var("FLUXER_MARKETING_ENDPOINT") .unwrap_or_else(|_| format!("{public_url}/marketing")); @@ -260,8 +378,13 @@ fn task_table_for_availability( }); insert(DevTask { name: "api", - args: strings(&["pnpm", "--filter", "fluxer_api", "dev"]), - cwd: ROOT.clone(), + args: vec![ + api_tsx.display().to_string(), + "watch".to_owned(), + "--clear-screen=false".to_owned(), + "src/AppEntrypoint.ts".to_owned(), + ], + cwd: api_dir.clone(), env: vec![ ( "FLUXER_S3_PUBLIC_ENDPOINT".to_owned(), @@ -286,29 +409,26 @@ fn task_table_for_availability( }); insert(DevTask { name: "worker", - args: strings(&[ - "pnpm", - "--filter", - "fluxer_api", - "exec", - "tsx", - "watch", - "--clear-screen=false", - "src/WorkerEntrypoint.ts", - ]), - cwd: ROOT.clone(), + args: vec![ + api_tsx.display().to_string(), + "watch".to_owned(), + "--clear-screen=false".to_owned(), + "src/WorkerEntrypoint.ts".to_owned(), + ], + cwd: api_dir, env: Vec::new(), }); insert(DevTask { name: "app", - args: strings(&["pnpm", "--filter", "fluxer_app", "dev"]), - cwd: ROOT.clone(), + args: vec![ + app_dev_server.display().to_string(), + "app-dev-server".to_owned(), + ], + cwd: app_dir, env: vec![ + ("TOKIO_WORKER_THREADS".to_owned(), Some("4".to_owned())), + ("RAYON_NUM_THREADS".to_owned(), Some("4".to_owned())), ("FLUXER_APP_DEV_PORT".to_owned(), Some(APP_PORT.to_string())), - ( - "FLUXER_APP_SKIP_I18N_COMPILE".to_owned(), - Some("true".to_owned()), - ), ( "FLUXER_STATIC_CDN_ENDPOINT".to_owned(), Some(public_url.clone()), @@ -584,27 +704,39 @@ async fn wait_for_search_backend_if_needed(selected: &[String]) -> Result<()> { return Ok(()); } let env = merged_env(None, true)?; + let backend = SearchBackend::from_env(&env)?; let search_url = env .get("FLUXER_SEARCH_URL") - .cloned() - .unwrap_or_else(|| default_search_url(&env)); - wait_http(search_backend_label(&env), &search_url, 120).await + .filter(|url| !url.trim().is_empty()) + .context("Missing FLUXER_SEARCH_URL for the selected search backend")?; + wait_http(backend.label(), search_url, 120).await } -fn default_search_url(env: &BTreeMap) -> String { - if env - .get("FLUXER_SEARCH_ENGINE") - .is_some_and(|engine| engine == "meilisearch") - { - return "http://127.0.0.1:7700".to_owned(); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SearchBackend { + Elasticsearch, + Meilisearch, +} + +impl SearchBackend { + fn from_env(env: &BTreeMap) -> Result { + match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) { + Some("elasticsearch") => Ok(Self::Elasticsearch), + Some("meilisearch") => Ok(Self::Meilisearch), + Some(engine) => bail!( + "Unsupported FLUXER_SEARCH_ENGINE value {engine:?}; expected `meilisearch` or `elasticsearch`" + ), + None => { + bail!("Missing FLUXER_SEARCH_ENGINE; expected `meilisearch` or `elasticsearch`") + } + } } - "http://127.0.0.1:9200".to_owned() -} -fn search_backend_label(env: &BTreeMap) -> &'static str { - match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) { - Some("meilisearch") => "Meilisearch", - _ => "Elasticsearch", + fn label(self) -> &'static str { + match self { + Self::Elasticsearch => "Elasticsearch", + Self::Meilisearch => "Meilisearch", + } } } @@ -627,9 +759,12 @@ fn public_url() -> String { .unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}")) } -fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> { +async fn ensure_js_dependencies_if_needed( + selected: &[String], + shutdown: &mut ShutdownSignal, +) -> Result> { if selected_needs_js_dependency_preflight(selected) { - run_command( + match run_command_interruptible( &["pnpm", "install", "--frozen-lockfile"], RunOptions { env: PNPM_INSTALL_ENV @@ -638,11 +773,17 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> { .collect(), ..RunOptions::default() }, - )?; + shutdown, + ) + .await? + { + AwaitOutcome::Completed(_) => {} + AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)), + } } if selected.iter().any(|name| name == "marketing") { let marketing_dir = ROOT.join("fluxer_marketing"); - run_command( + match run_command_interruptible( &["pnpm", "install", "--frozen-lockfile"], RunOptions { cwd: &marketing_dir, @@ -652,9 +793,15 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> { .collect(), ..RunOptions::default() }, - )?; + shutdown, + ) + .await? + { + AwaitOutcome::Completed(_) => {} + AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)), + } } - Ok(()) + Ok(AwaitOutcome::Completed(())) } fn selected_needs_js_dependency_preflight(selected: &[String]) -> bool { @@ -689,14 +836,7 @@ fn start_task(task: &DevTask) -> Result { .envs(env) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - #[cfg(unix)] - unsafe { - use std::os::unix::process::CommandExt; - command.pre_exec(|| { - libc::setsid(); - Ok(()) - }); - } + configure_process_group(&mut command); let mut child = command.spawn()?; if let Some(stdout) = child.stdout.take() { let label = task.name.to_owned(); @@ -718,13 +858,16 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> { .len() .checked_sub(1) .expect("services process was just pushed"); + let deadline = Instant::now() + Duration::from_secs(timeout); for spec in rust_services() { for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] { - check_startup_processes(processes, services_index, spec.name, mode)?; - wait_http( - &format!("Rust service {}:{mode}", spec.name), + let name = format!("Rust service {}:{mode}", spec.name); + wait_http_with_startup_checks_until( + processes, + services_index, + &name, &format!("http://{LOOPBACK_HOST}:{port}/_health"), - timeout, + deadline, ) .await?; } @@ -732,6 +875,25 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> { Ok(()) } +async fn wait_for_gateway(processes: &mut [Child], port: u16) -> Result<()> { + let timeout = std::env::var("FLUXER_DEV_GATEWAY_READY_TIMEOUT") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(180); + let gateway_index = processes + .len() + .checked_sub(1) + .expect("gateway process was just pushed"); + wait_http_with_startup_checks( + processes, + gateway_index, + "Gateway", + &format!("http://{LOOPBACK_HOST}:{port}/_health/ready"), + timeout, + ) + .await +} + async fn wait_for_api(processes: &mut [Child]) -> Result<()> { let timeout = std::env::var("FLUXER_DEV_API_READY_TIMEOUT") .ok() @@ -827,7 +989,7 @@ async fn wait_http_for_dev_tasks( while Instant::now() < deadline { check_dev_startup_processes(processes, name)?; match client.get(url).send().await { - Ok(response) if response.status().as_u16() < 500 => { + Ok(response) if response.status().is_success() => { println!("{name} is reachable at {url}"); return Ok(()); } @@ -849,16 +1011,32 @@ async fn wait_http_with_startup_checks( name: &str, url: &str, timeout_secs: u64, +) -> Result<()> { + wait_http_with_startup_checks_until( + processes, + watched_index, + name, + url, + Instant::now() + Duration::from_secs(timeout_secs), + ) + .await +} + +async fn wait_http_with_startup_checks_until( + processes: &mut [Child], + watched_index: usize, + name: &str, + url: &str, + deadline: Instant, ) -> Result<()> { let client = reqwest::Client::builder() .timeout(Duration::from_secs(5)) .build()?; - let deadline = Instant::now() + Duration::from_secs(timeout_secs); let mut last_error = None; while Instant::now() < deadline { check_startup_task_processes(processes, watched_index, name)?; match client.get(url).send().await { - Ok(response) if response.status().as_u16() < 500 => { + Ok(response) if response.status().is_success() => { println!("{name} is reachable at {url}"); return Ok(()); } @@ -901,35 +1079,6 @@ fn check_startup_task_processes( Ok(()) } -fn check_startup_processes( - processes: &mut [Child], - services_index: usize, - service_name: &str, - mode: &str, -) -> Result<()> { - for (index, process) in processes.iter_mut().enumerate() { - if let Some(status) = process.try_wait()? { - let code = status.code().unwrap_or(1); - if index == services_index { - bail!( - "Rust service supervisor exited with status {code} before {service_name}:{mode} became ready" - ); - } - bail!( - "Dev task exited with status {code} before Rust service {service_name}:{mode} became ready" - ); - } - } - Ok(()) -} - -fn prefix_output(label: &str, reader: impl std::io::Read) { - use std::io::{BufRead, BufReader}; - for line in BufReader::new(reader).lines().map_while(|line| line.ok()) { - println!("[{label}] {line}"); - } -} - #[allow(dead_code)] fn _cwd_is_root(path: &Path) -> bool { path == ROOT.as_path() @@ -940,17 +1089,17 @@ mod tests { use super::*; #[test] - fn default_tasks_keep_legacy_order() { + fn default_tasks_use_lightweight_gateway() { assert_eq!( DEFAULT_TASKS, &[ "proxy", "services", "media", - "gateway", "marketing", "admin", "api", + "gateway-single", "worker", "app", "app-proxy" diff --git a/tools/dev/src/gateway.rs b/tools/dev/src/gateway.rs index a06644bb7..db8cf2956 100644 --- a/tools/dev/src/gateway.rs +++ b/tools/dev/src/gateway.rs @@ -1,18 +1,20 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use crate::gateway_reload::{ - ArtifactState, changed_artifacts, hot_reload_enabled, hot_reload_modules, snapshot_artifacts, - spawn_source_watcher, -}; use crate::paths::{DEV_GATEWAY_DIR, GATEWAY_CONFIG_DIR, ROOT}; use crate::proc::{ - RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env, - restart_budget_exceeded, run_command, + AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown, + configure_process_group, force_kill_process_group, format_command, merged_env, + pending_shutdown, prefix_output, process_group_running, restart_budget_exceeded, + run_command_interruptible, stop_process_group, terminate_process_group, }; use anyhow::{Context, Result, bail}; +use sha2::{Digest, Sha256}; +#[cfg(target_os = "linux")] +use std::collections::BTreeSet; use std::collections::VecDeque; use std::env; use std::fs; +use std::io::Read; use std::net::{SocketAddr, TcpListener}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; @@ -47,9 +49,14 @@ const GATEWAY_COMPILE_COMMAND: &[&str] = &[ "--step", "compile", ]; +const GATEWAY_DEPENDENCY_INPUTS: &[&str] = &["rebar.config", "rebar.config.script", "rebar.lock"]; +const GATEWAY_DEPENDENCY_INPUT_MAX_BYTES: u64 = 4 * 1024 * 1024; +const GATEWAY_DEPENDENCY_STAMP_FILE: &str = "rebar-dependencies.sha256"; +const GATEWAY_DEPENDENCY_STAMP_MAX_BYTES: u64 = 128; const NODE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); const NODE_RESTART_PORT_WAIT: Duration = Duration::from_secs(75); -const STOP_GRACE_PERIOD: Duration = Duration::from_secs(10); +const NESTED_STOP_GRACE_PERIOD: Duration = Duration::from_secs(5); +const ROOT_STOP_GRACE_PERIOD: Duration = Duration::from_secs(15); #[derive(Debug, Clone, PartialEq, Eq)] pub struct GatewayNode { @@ -128,33 +135,42 @@ fn remove_stale_gateway_config() -> Result<()> { Ok(()) } -pub fn run_gateway() -> Result<()> { - setup_gateway_config()?; - compile_gateway()?; - let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?; - let env = merged_env(None, true)?; - println!("$ {}", format_command(&command)); - - #[cfg(unix)] - { - use std::os::unix::process::CommandExt; - let err = Command::new(&command[0]) - .args(&command[1..]) - .current_dir(gateway_dir()) - .env_clear() - .envs(env) - .exec(); - bail!("failed to exec gateway: {err}"); +pub async fn run_gateway() -> Result { + let mut shutdown = ShutdownSignal::new()?; + let node_names = std::collections::HashSet::from([env::var("FLUXER_ERLANG_NODE_NAME") + .unwrap_or_else(|_| "fluxer_gateway@127.0.0.1".to_owned())]); + cleanup_orphaned_gateway_processes(&node_names).await?; + if let Some(signal) = pending_shutdown(&mut shutdown).await { + println!("Received {signal}; stopping gateway startup..."); + return Ok(0); } - #[cfg(not(unix))] - { - let status = Command::new(&command[0]) - .args(&command[1..]) - .current_dir(gateway_dir()) - .env_clear() - .envs(env) - .status()?; - std::process::exit(status.code().unwrap_or(1)); + setup_gateway_config()?; + match compile_gateway_interruptible(&mut shutdown).await? { + AwaitOutcome::Completed(_) => {} + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping gateway startup..."); + return Ok(0); + } + } + let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?; + let command_refs = command.iter().map(String::as_str).collect::>(); + let working_dir = gateway_dir(); + let outcome = run_command_interruptible( + &command_refs, + RunOptions { + cwd: &working_dir, + ..RunOptions::default() + }, + &mut shutdown, + ) + .await?; + stop_idle_epmd(); + match outcome { + AwaitOutcome::Completed(output) => Ok(output.status.code().unwrap_or(1)), + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping gateway..."); + Ok(0) + } } } @@ -166,11 +182,31 @@ struct SupervisedNode { pub async fn run_gateway_cluster() -> Result { let nodes = build_gateway_cluster_nodes()?; - cleanup_orphaned_gateway_nodes(&nodes).await?; - wait_for_cluster_ports_available(&nodes).await?; - setup_gateway_cluster_config(&nodes)?; - compile_gateway()?; let mut shutdown = ShutdownSignal::new()?; + let node_names = nodes + .iter() + .map(GatewayNode::erlang_name) + .collect::>(); + cleanup_orphaned_gateway_processes(&node_names).await?; + if let Some(signal) = pending_shutdown(&mut shutdown).await { + println!("Received {signal}; stopping gateway startup..."); + return Ok(0); + } + match await_or_shutdown(&mut shutdown, wait_for_cluster_ports_available(&nodes)).await { + AwaitOutcome::Completed(result) => result?, + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping gateway startup..."); + return Ok(0); + } + } + setup_gateway_cluster_config(&nodes)?; + match compile_gateway_interruptible(&mut shutdown).await? { + AwaitOutcome::Completed(_) => {} + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping gateway startup..."); + return Ok(0); + } + } let static_peers = nodes .iter() .map(GatewayNode::erlang_name) @@ -179,57 +215,35 @@ pub async fn run_gateway_cluster() -> Result { let mut supervised = Vec::new(); print_gateway_cluster_topology(&nodes); for node in &nodes { + let child = match start_node(node, &static_peers) { + Ok(child) => child, + Err(error) => { + stop_supervised(&mut supervised); + return Err(error); + } + }; supervised.push(SupervisedNode { node: node.clone(), - child: start_node(node, &static_peers)?, + child, restarts: VecDeque::new(), }); } - let watcher = hot_reload_enabled().then(spawn_source_watcher); - if watcher.is_some() { - println!( - "Gateway hot reload enabled; watching fluxer_gateway sources (set FLUXER_DEV_GATEWAY_HOT_RELOAD=false to disable)" - ); - } - let mut compile: Option<(tokio::task::JoinHandle, ArtifactState)> = None; - let mut compile_queued = false; loop { - if let Err(error) = restart_exited_nodes(&mut supervised, &static_peers).await { - stop_supervised(&mut supervised); - return Err(error); - } - if let Some(receiver) = &watcher { - while receiver.try_recv().is_ok() { - compile_queued = true; - } - } - if compile_queued && compile.is_none() { - compile_queued = false; - println!("Gateway sources changed; recompiling for hot reload..."); - let before = snapshot_artifacts(); - compile = Some(( - tokio::task::spawn_blocking(compile_gateway_for_reload), - before, - )); - } - if compile - .as_ref() - .is_some_and(|(handle, _)| handle.is_finished()) + match await_or_shutdown( + &mut shutdown, + restart_exited_nodes(&mut supervised, &static_peers), + ) + .await { - let (handle, before) = compile.take().expect("compile task present"); - match handle.await { - Ok(true) => { - if let Err(error) = - apply_hot_reload(&mut supervised, &before, &static_peers).await - { - stop_supervised(&mut supervised); - return Err(error); - } - } - Ok(false) => println!( - "Gateway compile failed; hot reload skipped (fix the errors and save again)" - ), - Err(error) => println!("Gateway compile task failed: {error}"), + AwaitOutcome::Completed(Ok(())) => {} + AwaitOutcome::Completed(Err(error)) => { + stop_supervised(&mut supervised); + return Err(error); + } + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping gateway cluster..."); + stop_supervised(&mut supervised); + return Ok(0); } } tokio::select! { @@ -243,24 +257,116 @@ pub async fn run_gateway_cluster() -> Result { } } -fn compile_gateway() -> Result<()> { - run_command(GATEWAY_COMPILE_COMMAND, RunOptions::default()).map(drop) +async fn compile_gateway_interruptible( + shutdown: &mut ShutdownSignal, +) -> Result> { + let dependency_stamp = prepare_gateway_compile()?; + let outcome = + run_command_interruptible(GATEWAY_COMPILE_COMMAND, RunOptions::default(), shutdown).await?; + if matches!(&outcome, AwaitOutcome::Completed(_)) { + write_gateway_dependency_stamp(&dependency_stamp)?; + } + Ok(outcome) } -fn compile_gateway_for_reload() -> bool { - run_command( - GATEWAY_COMPILE_COMMAND, - RunOptions { - check: false, - ..RunOptions::default() - }, - ) - .map(|output| output.status.success()) - .unwrap_or(false) +fn prepare_gateway_compile() -> Result { + let dependency_stamp = gateway_dependency_stamp()?; + if gateway_dependency_stamp_matches(&dependency_stamp)? { + return Ok(dependency_stamp); + } + let build_dir = gateway_dir().join("_build/default"); + println!( + "Gateway dependency inputs changed; removing stale {}", + build_dir.display() + ); + match fs::remove_dir_all(&build_dir) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| format!("failed to remove {}", build_dir.display())); + } + } + Ok(dependency_stamp) } -fn cluster_cookie() -> String { - env::var("FLUXER_ERLANG_COOKIE").unwrap_or_else(|_| GATEWAY_CLUSTER_COOKIE.to_owned()) +fn gateway_dependency_stamp() -> Result { + let mut hasher = Sha256::new(); + let mut total_bytes = 0_u64; + for input in GATEWAY_DEPENDENCY_INPUTS { + let path = gateway_dir().join(input); + let metadata = fs::metadata(&path) + .with_context(|| format!("failed to read metadata for {}", path.display()))?; + if !metadata.is_file() { + bail!("gateway dependency input is not a file: {}", path.display()); + } + total_bytes = total_bytes + .checked_add(metadata.len()) + .context("gateway dependency input byte count overflowed")?; + if total_bytes > GATEWAY_DEPENDENCY_INPUT_MAX_BYTES { + bail!( + "gateway dependency inputs exceed {} bytes", + GATEWAY_DEPENDENCY_INPUT_MAX_BYTES + ); + } + let mut bytes = Vec::with_capacity(metadata.len() as usize); + fs::File::open(&path) + .with_context(|| format!("failed to open {}", path.display()))? + .take(metadata.len() + 1) + .read_to_end(&mut bytes) + .with_context(|| format!("failed to read {}", path.display()))?; + if bytes.len() as u64 != metadata.len() { + bail!( + "gateway dependency input changed while reading: {}", + path.display() + ); + } + hasher.update((input.len() as u64).to_le_bytes()); + hasher.update(input.as_bytes()); + hasher.update(metadata.len().to_le_bytes()); + hasher.update(&bytes); + } + Ok(bytes_to_lower_hex(&hasher.finalize())) +} + +fn bytes_to_lower_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(HEX[(byte >> 4) as usize] as char); + output.push(HEX[(byte & 0x0f) as usize] as char); + } + output +} + +fn gateway_dependency_stamp_matches(expected: &str) -> Result { + let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE); + let file = match fs::File::open(&path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => { + return Err(error).with_context(|| format!("failed to open {}", path.display())); + } + }; + let mut bytes = Vec::new(); + file.take(GATEWAY_DEPENDENCY_STAMP_MAX_BYTES + 1) + .read_to_end(&mut bytes) + .with_context(|| format!("failed to read {}", path.display()))?; + if bytes.len() as u64 > GATEWAY_DEPENDENCY_STAMP_MAX_BYTES { + bail!( + "gateway dependency stamp exceeds {} bytes: {}", + GATEWAY_DEPENDENCY_STAMP_MAX_BYTES, + path.display() + ); + } + Ok(bytes == format!("{expected}\n").as_bytes()) +} + +fn write_gateway_dependency_stamp(dependency_stamp: &str) -> Result<()> { + fs::create_dir_all(DEV_GATEWAY_DIR.as_path()) + .with_context(|| format!("failed to create {}", DEV_GATEWAY_DIR.display()))?; + let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE); + fs::write(&path, format!("{dependency_stamp}\n")) + .with_context(|| format!("failed to write {}", path.display())) } async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: &str) -> Result<()> { @@ -285,18 +391,7 @@ async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: & } async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result<()> { - if entry.child.try_wait()?.is_none() { - terminate_process(&mut entry.child); - let deadline = Instant::now() + NODE_SHUTDOWN_TIMEOUT; - while entry.child.try_wait()?.is_none() { - if Instant::now() >= deadline { - let _ = entry.child.kill(); - let _ = entry.child.wait(); - break; - } - sleep(Duration::from_millis(100)).await; - } - } + stop_process_group(&mut entry.child, NODE_SHUTDOWN_TIMEOUT).await; wait_for_ports_available_until( std::slice::from_ref(&entry.node), Instant::now() + NODE_RESTART_PORT_WAIT, @@ -306,79 +401,13 @@ async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result< Ok(()) } -async fn apply_hot_reload( - supervised: &mut [SupervisedNode], - before: &ArtifactState, - static_peers: &str, -) -> Result<()> { - let after = snapshot_artifacts(); - let diff = changed_artifacts(before, &after); - if diff.nifs_changed { - println!("Gateway native NIF artifacts changed; rolling restart of all gateway nodes..."); - for entry in supervised.iter_mut() { - println!("[gateway:{}] restarting for NIF reload", entry.node.name()); - restart_node(entry, static_peers).await?; - } - println!("Gateway rolling restart complete"); - return Ok(()); - } - if diff.modules.is_empty() { - println!("Gateway compile finished; no module changes to reload"); - return Ok(()); - } - println!( - "Hot reloading {} gateway module(s) across {} node(s): {}", - diff.modules.len(), - supervised.len(), - diff.modules.join(", ") - ); - let nodes = supervised - .iter() - .map(|entry| entry.node.clone()) - .collect::>(); - let modules = diff.modules.clone(); - let cookie = cluster_cookie(); - let outcome = - tokio::task::spawn_blocking(move || hot_reload_modules(&nodes, &modules, &cookie)).await; - match outcome { - Ok(Ok(outcome)) if outcome.failed_nodes.is_empty() => { - println!( - "Gateway hot reload complete ({} module(s) live)", - diff.modules.len() - ); - } - Ok(Ok(outcome)) => { - for entry in supervised.iter_mut() { - if !outcome.failed_nodes.contains(&entry.node.erlang_name()) { - continue; - } - if entry.child.try_wait()?.is_some() { - continue; - } - println!( - "[gateway:{}] hot reload failed; restarting node to pick up new code", - entry.node.name() - ); - restart_node(entry, static_peers).await?; - } - } - Ok(Err(error)) => { - println!("Gateway hot reload failed: {error}; rolling restart of all gateway nodes"); - for entry in supervised.iter_mut() { - restart_node(entry, static_peers).await?; - } - } - Err(error) => println!("Gateway hot reload task failed: {error}"), - } - Ok(()) -} - fn stop_supervised(supervised: &mut [SupervisedNode]) { let mut children = supervised .iter_mut() .map(|entry| &mut entry.child) .collect::>(); stop_child_processes(&mut children); + stop_idle_epmd(); } pub fn build_gateway_cluster_nodes() -> Result> { @@ -492,18 +521,20 @@ fn can_bind_tcp_port(port: u16) -> bool { } #[cfg(target_os = "linux")] -async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> { - let leaders = orphaned_gateway_leaders(nodes)?; +async fn cleanup_orphaned_gateway_processes( + node_names: &std::collections::HashSet, +) -> Result<()> { + let leaders = orphaned_gateway_leaders(node_names)?; if leaders.is_empty() { return Ok(()); } let pids = leaders.iter().map(|leader| leader.pid).collect::>(); println!( - "Stopping orphaned gateway cluster node process group(s): {}", + "Stopping orphaned gateway process group(s): {}", format_pids(&pids) ); - let term_failed = signal_process_groups(&pids, libc::SIGTERM); + let term_failed = signal_process_groups(&leaders, libc::SIGTERM); if !term_failed.is_empty() { println!( "SIGTERM delivery failed for orphaned gateway pid(s): {}", @@ -518,7 +549,12 @@ async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> { return Ok(()); } if Instant::now() >= deadline { - let kill_failed = signal_process_groups(&remaining, libc::SIGKILL); + let remaining_leaders = leaders + .iter() + .filter(|leader| remaining.contains(&leader.pid)) + .cloned() + .collect::>(); + let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL); sleep(Duration::from_millis(200)).await; let survivors = surviving_gateway_group_pids(&leaders)?; if survivors.is_empty() { @@ -556,24 +592,95 @@ fn format_pids(pids: &[i32]) -> String { } #[cfg(not(target_os = "linux"))] -async fn cleanup_orphaned_gateway_nodes(_nodes: &[GatewayNode]) -> Result<()> { +async fn cleanup_orphaned_gateway_processes( + _node_names: &std::collections::HashSet, +) -> Result<()> { Ok(()) } #[cfg(target_os = "linux")] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] struct GatewayLeader { + pid: i32, + members: Vec, +} + +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct GatewayProcessIdentity { pid: i32, starttime: u64, } #[cfg(target_os = "linux")] -fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result> { - let node_names = nodes +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct GatewayProcess { + pid: i32, + ppid: i32, + pgid: i32, + state: char, + starttime: u64, +} + +#[cfg(target_os = "linux")] +fn orphaned_gateway_leaders( + node_names: &std::collections::HashSet, +) -> Result> { + let current_exe = std::env::current_exe().context("failed to resolve fluxer-dev executable")?; + let processes = gateway_process_snapshot()?; + let mut owned_pids = BTreeSet::new(); + for process in &processes { + if process.ppid != 1 || proc_stat_state_is_dead(process.state) { + continue; + } + let args = proc_cmdline(process.pid); + let orphaned_supervisor = cmdline_is_gateway_supervisor(&args, ¤t_exe); + let orphaned_node = (cmdline_has_gateway_node(&args, node_names) + || cmdline_has_managed_gateway_config(&args)) + && fs::read_link(format!("/proc/{}/cwd", process.pid)) + .ok() + .as_deref() + == Some(gateway_dir().as_path()); + if orphaned_supervisor || orphaned_node { + owned_pids.insert(process.pid); + } + } + loop { + let previous_count = owned_pids.len(); + for process in &processes { + if owned_pids.contains(&process.ppid) { + owned_pids.insert(process.pid); + } + } + if owned_pids.len() == previous_count { + break; + } + } + let group_pids = processes .iter() - .map(GatewayNode::erlang_name) - .collect::>(); - let mut leaders = Vec::new(); + .filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1) + .map(|process| process.pgid) + .collect::>(); + let leaders = group_pids + .into_iter() + .map(|pid| GatewayLeader { + pid, + members: processes + .iter() + .filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid) + .map(|process| GatewayProcessIdentity { + pid: process.pid, + starttime: process.starttime, + }) + .collect(), + }) + .collect::>(); + Ok(leaders) +} + +#[cfg(target_os = "linux")] +fn gateway_process_snapshot() -> Result> { + let mut processes = Vec::new(); for entry in fs::read_dir("/proc").context("failed to read /proc")? { let Ok(entry) = entry else { continue; @@ -585,18 +692,39 @@ fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result> else { continue; }; - if !cmdline_has_gateway_node(&proc_cmdline(pid), &node_names) { + let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { continue; - } - if proc_parent_pid(pid) != Some(1) { + }; + let Some((state, ppid, pgid, starttime)) = parse_proc_stat_process(&stat) else { continue; - } - if let Some(starttime) = proc_stat_starttime(pid) { - leaders.push(GatewayLeader { pid, starttime }); - } + }; + processes.push(GatewayProcess { + pid, + ppid, + pgid, + state, + starttime, + }); } - leaders.sort_unstable_by_key(|leader| leader.pid); - Ok(leaders) + Ok(processes) +} + +#[cfg(target_os = "linux")] +fn cmdline_is_gateway_supervisor(args: &[String], current_exe: &Path) -> bool { + args.first().map(Path::new) == Some(current_exe) + && args.get(1).map(String::as_str) == Some("gateway") + && match args.get(2).map(String::as_str) { + None | Some("cluster" | "single") => args.len() <= 3, + Some(_) => false, + } +} + +#[cfg(target_os = "linux")] +fn cmdline_has_managed_gateway_config(args: &[String]) -> bool { + args.windows(2).any(|pair| { + matches!(pair[0].as_str(), "-config" | "-args_file") + && Path::new(&pair[1]).starts_with(DEV_GATEWAY_DIR.as_path()) + }) } #[cfg(any(target_os = "linux", test))] @@ -618,30 +746,6 @@ fn proc_cmdline(pid: i32) -> Vec { .collect() } -#[cfg(target_os = "linux")] -fn proc_parent_pid(pid: i32) -> Option { - fs::read_to_string(format!("/proc/{pid}/status")) - .ok()? - .lines() - .find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok()) -} - -#[cfg(target_os = "linux")] -fn process_exists(pid: i32) -> bool { - assert!(pid > 0); - match proc_stat_state_and_pgid(pid) { - Some((state, _pgid)) => !proc_stat_state_is_dead(state), - None => false, - } -} - -#[cfg(target_os = "linux")] -fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> { - assert!(pid > 0); - let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; - parse_proc_stat_state_and_pgid(&stat) -} - #[cfg(any(target_os = "linux", test))] fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> { let (_, after_comm) = stat.rsplit_once(')')?; @@ -653,10 +757,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> { } #[cfg(target_os = "linux")] -fn proc_stat_starttime(pid: i32) -> Option { - assert!(pid > 0); - let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; - parse_proc_stat_starttime(&stat) +fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> { + let (_, after_comm) = stat.rsplit_once(')')?; + let mut fields = after_comm.split_ascii_whitespace(); + let state = fields.next()?.chars().next()?; + let ppid = fields.next()?.parse().ok()?; + let pgid = fields.next()?.parse().ok()?; + Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?)) } #[cfg(any(target_os = "linux", test))] @@ -672,62 +779,43 @@ fn proc_stat_state_is_dead(state: char) -> bool { #[cfg(target_os = "linux")] fn surviving_gateway_group_pids(leaders: &[GatewayLeader]) -> Result> { - let leader_pids = leaders + Ok(leaders .iter() + .filter(|leader| gateway_group_has_owned_member(leader)) .map(|leader| leader.pid) - .collect::>(); - let mut survivors = Vec::new(); - for entry in fs::read_dir("/proc").context("failed to read /proc")? { - let Ok(entry) = entry else { - continue; - }; - let Some(pid) = entry - .file_name() - .to_str() - .and_then(|name| name.parse::().ok()) - else { - continue; - }; - let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { - continue; - }; - let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else { - continue; - }; - if proc_stat_state_is_dead(state) { - continue; - } - if leader_pids.contains(&pgid) { - survivors.push(pid); - continue; - } - let leader = leaders.iter().find(|leader| leader.pid == pid); - if let Some(leader) = leader - && parse_proc_stat_starttime(&stat) == Some(leader.starttime) - { - survivors.push(pid); - } - } - survivors.sort_unstable(); - Ok(survivors) + .collect()) } #[cfg(target_os = "linux")] -fn signal_process_groups(pids: &[i32], signal: i32) -> Vec { +fn gateway_group_has_owned_member(leader: &GatewayLeader) -> bool { + leader.members.iter().any(|member| { + let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else { + return false; + }; + let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else { + return false; + }; + !proc_stat_state_is_dead(state) + && pgid == leader.pid + && parse_proc_stat_starttime(&stat) == Some(member.starttime) + }) +} + +#[cfg(target_os = "linux")] +fn signal_process_groups(leaders: &[GatewayLeader], signal: i32) -> Vec { assert!(signal == libc::SIGTERM || signal == libc::SIGKILL); - let mut failed = Vec::with_capacity(pids.len()); - for pid in pids { - assert!(*pid > 0); - let group_result = unsafe { libc::kill(-pid, signal) }; + let mut failed = Vec::with_capacity(leaders.len()); + for leader in leaders { + assert!(leader.pid > 1); + if !gateway_group_has_owned_member(leader) { + continue; + } + let group_result = unsafe { libc::kill(-leader.pid, signal) }; if group_result == 0 { continue; } - let direct_result = unsafe { libc::kill(*pid, signal) }; - if direct_result == 0 { - continue; - } - if process_exists(*pid) { - failed.push(*pid); + if gateway_group_has_owned_member(leader) { + failed.push(leader.pid); } } failed @@ -812,14 +900,7 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result { .envs(env) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - #[cfg(unix)] - unsafe { - use std::os::unix::process::CommandExt; - child_command.pre_exec(|| { - libc::setsid(); - Ok(()) - }); - } + configure_process_group(&mut child_command); let mut child = child_command.spawn()?; if let Some(stdout) = child.stdout.take() { let label = node.name(); @@ -832,30 +913,30 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result { Ok(child) } -fn prefix_output(label: &str, reader: impl std::io::Read) { - use std::io::{BufRead, BufReader}; - for line in BufReader::new(reader).lines().map_while(|line| line.ok()) { - println!("[{label}] {line}"); - } -} - pub fn stop_processes(processes: &mut [Child]) { let mut children = processes.iter_mut().collect::>(); - stop_child_processes(&mut children); + stop_child_processes_with_grace(&mut children, ROOT_STOP_GRACE_PERIOD); + stop_idle_epmd(); } pub fn stop_child_processes(processes: &mut [&mut Child]) { + stop_child_processes_with_grace(processes, NESTED_STOP_GRACE_PERIOD); +} + +fn stop_child_processes_with_grace(processes: &mut [&mut Child], grace_period: Duration) { for process in processes.iter_mut() { - if process.try_wait().ok().flatten().is_some() { - continue; + if let Err(error) = terminate_process_group(process) { + eprintln!( + "Failed to terminate process group {}: {error}", + process.id() + ); } - terminate_process(process); } - let deadline = Instant::now() + STOP_GRACE_PERIOD; + let deadline = Instant::now() + grace_period; loop { - let all_exited = processes - .iter_mut() - .all(|process| process.try_wait().ok().flatten().is_some()); + let all_exited = processes.iter_mut().all(|process| { + process.try_wait().ok().flatten().is_some() && !process_group_running(process) + }); if all_exited { return; } @@ -865,38 +946,30 @@ pub fn stop_child_processes(processes: &mut [&mut Child]) { std::thread::sleep(Duration::from_millis(100)); } for process in processes { - if process.try_wait().ok().flatten().is_some() { - continue; + if process.try_wait().ok().flatten().is_none() || process_group_running(process) { + force_kill_process_group(process); } - force_kill_process(process); } } -#[cfg(unix)] -fn terminate_process(process: &mut Child) { - unsafe { - libc::kill(-(process.id() as i32), libc::SIGTERM); +fn stop_idle_epmd() { + let output = match Command::new("epmd").arg("-kill").output() { + Ok(output) => output, + Err(error) => { + eprintln!("Failed to stop Erlang port mapper: {error}"); + return; + } + }; + if output.status.success() { + return; } -} - -#[cfg(not(unix))] -fn terminate_process(process: &mut Child) { - let _ = process.kill(); -} - -#[cfg(unix)] -fn force_kill_process(process: &mut Child) { - unsafe { - libc::kill(-(process.id() as i32), libc::SIGKILL); + let mut message = output.stdout; + message.extend(output.stderr); + let message = String::from_utf8_lossy(&message); + if message.contains("Cannot connect to local epmd") { + return; } - let _ = process.kill(); - let _ = process.wait(); -} - -#[cfg(not(unix))] -fn force_kill_process(process: &mut Child) { - let _ = process.kill(); - let _ = process.wait(); + eprintln!("Failed to stop Erlang port mapper: {}", message.trim()); } pub fn build_gateway_command(config_dir: &Path) -> Result> { diff --git a/tools/dev/src/gateway_reload.rs b/tools/dev/src/gateway_reload.rs deleted file mode 100644 index ee9f694a3..000000000 --- a/tools/dev/src/gateway_reload.rs +++ /dev/null @@ -1,420 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -use crate::gateway::{GatewayNode, gateway_dir}; -use anyhow::{Context, Result, bail}; -use std::collections::{BTreeMap, BTreeSet}; -use std::env; -use std::fs; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command, Stdio}; -use std::sync::mpsc::{Receiver, channel}; -use std::time::{Duration, Instant, SystemTime}; - -const WATCH_POLL_INTERVAL: Duration = Duration::from_millis(1000); -const WATCH_SETTLE_INTERVAL: Duration = Duration::from_millis(300); -const RELOAD_TIMEOUT: Duration = Duration::from_secs(60); -const WATCHED_SOURCE_EXTENSIONS: &[&str] = &["erl", "hrl", "src", "rs", "toml", "config"]; - -pub type FileState = BTreeMap; - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ArtifactState { - pub beams: FileState, - pub nifs: FileState, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ArtifactDiff { - pub modules: Vec, - pub nifs_changed: bool, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ReloadOutcome { - pub failed_nodes: Vec, -} - -pub fn hot_reload_enabled() -> bool { - env::var("FLUXER_DEV_GATEWAY_HOT_RELOAD") - .map(|value| !matches!(value.to_ascii_lowercase().as_str(), "0" | "false" | "no")) - .unwrap_or(true) -} - -pub fn spawn_source_watcher() -> Receiver<()> { - let (sender, receiver) = channel(); - std::thread::spawn(move || { - let mut previous = scan_sources(); - loop { - std::thread::sleep(WATCH_POLL_INTERVAL); - let mut current = scan_sources(); - if current == previous { - continue; - } - loop { - std::thread::sleep(WATCH_SETTLE_INTERVAL); - let settled = scan_sources(); - if settled == current { - break; - } - current = settled; - } - previous = current; - if sender.send(()).is_err() { - return; - } - } - }); - receiver -} - -fn scan_sources() -> FileState { - let dir = gateway_dir(); - let mut state = FileState::new(); - for root in [dir.join("src"), dir.join("include"), dir.join("native")] { - scan_tree(&mut state, &root); - } - for path in [ - dir.join("rebar.config"), - dir.join("rebar.config.script"), - dir.join("rebar.lock"), - ] { - record_file(&mut state, &path); - } - state -} - -fn scan_tree(state: &mut FileState, root: &Path) { - let Ok(entries) = fs::read_dir(root) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - let Ok(metadata) = entry.metadata() else { - continue; - }; - if metadata.is_dir() { - if !is_skipped_dir(&path) { - scan_tree(state, &path); - } - } else if is_watched_source(&path) - && let Ok(modified) = metadata.modified() - { - state.insert(path, (modified, metadata.len())); - } - } -} - -fn record_file(state: &mut FileState, path: &Path) { - if let Ok(metadata) = fs::metadata(path) - && let Ok(modified) = metadata.modified() - { - state.insert(path.to_path_buf(), (modified, metadata.len())); - } -} - -fn is_skipped_dir(path: &Path) -> bool { - path.file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name == "target" || name.starts_with('.')) -} - -fn is_watched_source(path: &Path) -> bool { - path.extension() - .and_then(|extension| extension.to_str()) - .is_some_and(|extension| WATCHED_SOURCE_EXTENSIONS.contains(&extension)) -} - -pub fn snapshot_artifacts() -> ArtifactState { - let mut state = ArtifactState::default(); - let lib_root = gateway_dir().join("_build/default/lib"); - if let Ok(entries) = fs::read_dir(&lib_root) { - for entry in entries.flatten() { - scan_artifact_dir(&mut state.beams, &entry.path().join("ebin"), "beam"); - } - } - scan_artifact_dir(&mut state.nifs, &gateway_dir().join("priv"), "so"); - state -} - -fn scan_artifact_dir(state: &mut FileState, dir: &Path, extension: &str) { - let Ok(entries) = fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path - .extension() - .is_some_and(|candidate| candidate == extension) - && let Ok(metadata) = entry.metadata() - && let Ok(modified) = metadata.modified() - { - state.insert(path, (modified, metadata.len())); - } - } -} - -pub fn changed_artifacts(before: &ArtifactState, after: &ArtifactState) -> ArtifactDiff { - let modules = after - .beams - .iter() - .filter(|(path, state)| before.beams.get(*path) != Some(*state)) - .filter_map(|(path, _)| Some(path.file_stem()?.to_str()?.to_owned())) - .collect::>() - .into_iter() - .collect(); - let nifs_changed = after - .nifs - .iter() - .any(|(path, state)| before.nifs.get(path) != Some(state)); - ArtifactDiff { - modules, - nifs_changed, - } -} - -pub fn build_reload_eval(nodes: &[String], modules: &[String]) -> String { - let node_list = nodes - .iter() - .map(|node| format!("'{node}'")) - .collect::>() - .join(","); - let module_list = modules - .iter() - .map(|module| format!("'{module}'")) - .collect::>() - .join(","); - format!( - "Nodes = [{node_list}], \ - Mods = [{module_list}], \ - Failed = [Node || Node <- Nodes, \ - case net_adm:ping(Node) of \ - pong -> \ - Errors = [Mod || Mod <- Mods, \ - begin \ - rpc:call(Node, code, purge, [Mod], 10000), \ - case rpc:call(Node, code, load_file, [Mod], 10000) of \ - {{module, Mod}} -> false; \ - Other -> \ - io:format(\"gateway_reload_error ~s ~s ~0p~n\", [Node, Mod, Other]), \ - true \ - end \ - end], \ - Errors =/= []; \ - pang -> \ - io:format(\"gateway_reload_node_down ~s~n\", [Node]), \ - true \ - end], \ - [io:format(\"gateway_reload_failed_node ~s~n\", [Failed1]) || Failed1 <- Failed], \ - io:format(\"gateway_reload_done~n\"), \ - halt(0)." - ) -} - -pub fn hot_reload_modules( - nodes: &[GatewayNode], - modules: &[String], - cookie: &str, -) -> Result { - assert!(!modules.is_empty()); - let node_names = nodes - .iter() - .map(GatewayNode::erlang_name) - .collect::>(); - let eval = build_reload_eval(&node_names, modules); - let reloader_name = format!("fluxer_dev_reload_{}@127.0.0.1", std::process::id()); - let mut child = Command::new("erl") - .args([ - "-hidden", - "-noshell", - "-name", - &reloader_name, - "-setcookie", - cookie, - "-eval", - &eval, - ]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn gateway reload shell")?; - let stdout = child.stdout.take().map(spawn_reader); - let stderr = child.stderr.take().map(spawn_reader); - let status = wait_with_deadline(&mut child, RELOAD_TIMEOUT)?; - let mut output = String::new(); - for handle in [stdout, stderr].into_iter().flatten() { - if let Ok(text) = handle.join() { - output.push_str(&text); - } - } - for line in output.lines().filter(|line| !line.trim().is_empty()) { - println!("[gateway:reload] {line}"); - } - if !status.success() { - bail!( - "gateway reload shell exited with status {}", - status.code().unwrap_or(1) - ); - } - if !output.lines().any(|line| line == "gateway_reload_done") { - bail!("gateway reload shell did not report completion"); - } - Ok(parse_reload_outcome(&output)) -} - -pub fn parse_reload_outcome(output: &str) -> ReloadOutcome { - let failed_nodes = output - .lines() - .filter_map(|line| line.strip_prefix("gateway_reload_failed_node ")) - .map(|node| node.trim().to_owned()) - .collect::>() - .into_iter() - .collect(); - ReloadOutcome { failed_nodes } -} - -fn spawn_reader(stream: impl Read + Send + 'static) -> std::thread::JoinHandle { - std::thread::spawn(move || { - let mut reader = stream; - let mut text = String::new(); - let mut bytes = Vec::new(); - if reader.read_to_end(&mut bytes).is_ok() { - text = String::from_utf8_lossy(&bytes).into_owned(); - } - text - }) -} - -fn wait_with_deadline(child: &mut Child, timeout: Duration) -> Result { - let deadline = Instant::now() + timeout; - loop { - if let Some(status) = child.try_wait()? { - return Ok(status); - } - if Instant::now() >= deadline { - let _ = child.kill(); - let _ = child.wait(); - bail!( - "gateway reload shell timed out after {}s", - timeout.as_secs() - ); - } - std::thread::sleep(Duration::from_millis(100)); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn file_state(entries: &[(&str, u64, u64)]) -> FileState { - entries - .iter() - .map(|(path, seconds, size)| { - ( - PathBuf::from(path), - ( - SystemTime::UNIX_EPOCH + Duration::from_secs(*seconds), - *size, - ), - ) - }) - .collect() - } - - #[test] - fn changed_artifacts_detects_new_and_modified_beams() { - let before = ArtifactState { - beams: file_state(&[("ebin/a.beam", 1, 10), ("ebin/b.beam", 1, 20)]), - nifs: FileState::new(), - }; - let after = ArtifactState { - beams: file_state(&[ - ("ebin/a.beam", 2, 10), - ("ebin/b.beam", 1, 20), - ("ebin/c.beam", 1, 30), - ]), - nifs: FileState::new(), - }; - let diff = changed_artifacts(&before, &after); - assert_eq!(diff.modules, vec!["a".to_owned(), "c".to_owned()]); - assert!(!diff.nifs_changed); - } - - #[test] - fn changed_artifacts_ignores_unchanged_state() { - let state = ArtifactState { - beams: file_state(&[("ebin/a.beam", 1, 10)]), - nifs: file_state(&[("priv/a_nif.so", 1, 10)]), - }; - let diff = changed_artifacts(&state, &state.clone()); - assert!(diff.modules.is_empty()); - assert!(!diff.nifs_changed); - } - - #[test] - fn changed_artifacts_flags_nif_changes() { - let before = ArtifactState { - beams: FileState::new(), - nifs: file_state(&[("priv/a_nif.so", 1, 10)]), - }; - let after = ArtifactState { - beams: FileState::new(), - nifs: file_state(&[("priv/a_nif.so", 2, 11)]), - }; - assert!(changed_artifacts(&before, &after).nifs_changed); - } - - #[test] - fn reload_eval_quotes_nodes_and_modules() { - let eval = build_reload_eval( - &["fluxer_gateway_websocket_1@127.0.0.1".to_owned()], - &["gateway_compress".to_owned(), "push".to_owned()], - ); - assert!(eval.contains("Nodes = ['fluxer_gateway_websocket_1@127.0.0.1']")); - assert!(eval.contains("Mods = ['gateway_compress','push']")); - assert!(eval.contains("halt(0).")); - } - - #[test] - fn reload_outcome_parses_failed_nodes() { - let output = "gateway_reload_error n1 mod {error,nofile}\n\ - gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\ - gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\ - gateway_reload_done\n"; - let outcome = parse_reload_outcome(output); - assert_eq!( - outcome.failed_nodes, - vec!["fluxer_gateway_guilds_2@127.0.0.1".to_owned()] - ); - } - - #[test] - fn reload_outcome_empty_when_no_failures() { - assert!( - parse_reload_outcome("gateway_reload_done\n") - .failed_nodes - .is_empty() - ); - } - - #[test] - fn watched_source_filter_accepts_known_extensions() { - assert!(is_watched_source(Path::new("src/push.erl"))); - assert!(is_watched_source(Path::new("include/gateway.hrl"))); - assert!(is_watched_source(Path::new("src/fluxer_gateway.app.src"))); - assert!(is_watched_source(Path::new("native/a_nif/src/lib.rs"))); - assert!(is_watched_source(Path::new("native/a_nif/Cargo.toml"))); - assert!(!is_watched_source(Path::new("ebin/push.beam"))); - assert!(!is_watched_source(Path::new("src/.push.erl.swp"))); - assert!(!is_watched_source(Path::new("native/a_nif/Cargo.lock"))); - } - - #[test] - fn skipped_dirs_cover_build_output_and_hidden_dirs() { - assert!(is_skipped_dir(Path::new("native/a_nif/target"))); - assert!(is_skipped_dir(Path::new("src/.git"))); - assert!(!is_skipped_dir(Path::new("src/gateway"))); - } -} diff --git a/tools/dev/src/lib.rs b/tools/dev/src/lib.rs index 11f54c421..32122d28b 100644 --- a/tools/dev/src/lib.rs +++ b/tools/dev/src/lib.rs @@ -7,7 +7,6 @@ pub mod dev; pub mod disclaim; pub mod env; pub mod gateway; -pub mod gateway_reload; pub mod manifest; pub mod media_external; pub mod media_proxy; diff --git a/tools/dev/src/main.rs b/tools/dev/src/main.rs index 822e6695a..4f3f0a2ca 100644 --- a/tools/dev/src/main.rs +++ b/tools/dev/src/main.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use anyhow::Result; +use anyhow::{Context, Result, bail}; use clap::{Args, Parser, Subcommand}; use fluxer_dev::cassandra::{ apply_schema, compute_diff, render_target_schema, verify_schema, write_diff_file, @@ -11,9 +11,19 @@ use fluxer_dev::desktop::{ }; use fluxer_dev::env::merge_default_env_with_current; use fluxer_dev::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL}; -use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT_LOCAL_ENV_FILE}; +use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE}; use std::collections::BTreeMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; + +const DEV_INFRA_SERVICES: &[&str] = &[ + "postgres", + "valkey", + "nats", + "livekit", + "meilisearch", + "mailpit", +]; #[derive(Debug, Parser)] #[command(name = "fluxer-dev")] @@ -34,6 +44,7 @@ enum Command { Proxy(ProxyArgs), Dev(DevArgs), RustServices(RustServicesArgs), + Infra(InfraArgs), Cassandra(CassandraArgs), Desktop(DesktopArgs), MediaProxy(MediaProxyArgs), @@ -44,8 +55,6 @@ enum Command { struct BootstrapArgs { #[arg(long)] skip_install: bool, - #[arg(long)] - skip_desktop_install: bool, } #[derive(Debug, Args)] @@ -76,6 +85,19 @@ struct RustServicesArgs { services: Vec, } +#[derive(Debug, Args)] +struct InfraArgs { + #[command(subcommand)] + command: InfraCommand, +} + +#[derive(Debug, Subcommand)] +enum InfraCommand { + Start, + Stop, + Status, +} + #[derive(Debug, Args)] struct CassandraArgs { #[command(subcommand)] @@ -193,10 +215,12 @@ async fn main() -> Result<()> { match cli.command { Command::Bootstrap(args) => { - fluxer_dev::bootstrap::bootstrap(args.skip_install, args.skip_desktop_install).await?; + fluxer_dev::bootstrap::bootstrap(args.skip_install).await?; } Command::PostStart => fluxer_dev::bootstrap::post_start().await?, - Command::Gateway(args) if args.mode == "single" => fluxer_dev::gateway::run_gateway()?, + Command::Gateway(args) if args.mode == "single" => { + std::process::exit(fluxer_dev::gateway::run_gateway().await?) + } Command::Gateway(_) => { std::process::exit(fluxer_dev::gateway::run_gateway_cluster().await?) } @@ -216,6 +240,7 @@ async fn main() -> Result<()> { Command::RustServices(args) => { std::process::exit(fluxer_dev::rust_services::run_rust_services(&args.services).await?) } + Command::Infra(args) => run_infra(args.command)?, Command::Cassandra(args) => match args.command { CassandraCommand::Diff { output } => { let diff = compute_diff(None).await?; @@ -315,3 +340,78 @@ fn apply_default_env() -> Result<()> { } Ok(()) } + +fn run_infra(command: InfraCommand) -> Result<()> { + let project = compose_project_name()?; + let compose_file = ROOT.join(".devcontainer/docker-compose.yml"); + let mut args = vec![ + "compose".to_owned(), + "--project-name".to_owned(), + project, + "-f".to_owned(), + compose_file.display().to_string(), + ]; + match command { + InfraCommand::Start => args.push("start".to_owned()), + InfraCommand::Stop => args.push("stop".to_owned()), + InfraCommand::Status => { + args.push("ps".to_owned()); + args.push("--all".to_owned()); + } + } + args.extend( + DEV_INFRA_SERVICES + .iter() + .map(|service| (*service).to_owned()), + ); + let status = ProcessCommand::new("docker") + .args(&args) + .status() + .context("failed to run Docker Compose for the dev infrastructure")?; + if !status.success() { + bail!("Docker Compose dev infrastructure command failed with {status}"); + } + Ok(()) +} + +fn compose_project_name() -> Result { + if Path::new("/.dockerenv").exists() { + let container = std::fs::read_to_string("/etc/hostname") + .context("failed to read the current devcontainer hostname")?; + let container = container.trim(); + if container.is_empty() + || !container + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-')) + { + bail!("Current devcontainer hostname is invalid"); + } + let output = ProcessCommand::new("docker") + .args([ + "inspect", + "--format", + "{{ index .Config.Labels \"com.docker.compose.project\" }}", + container, + ]) + .output() + .context("failed to inspect the current devcontainer Compose project")?; + if !output.status.success() { + bail!( + "Could not discover the current devcontainer Compose project: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + let project = String::from_utf8(output.stdout) + .context("devcontainer Compose project label is not valid UTF-8")? + .trim() + .to_owned(); + if project.is_empty() || project == "" { + bail!("Current container has no com.docker.compose.project label"); + } + return Ok(project); + } + std::env::var("COMPOSE_PROJECT_NAME") + .ok() + .filter(|project| !project.trim().is_empty()) + .context("COMPOSE_PROJECT_NAME must be set when managing dev infrastructure from the host") +} diff --git a/tools/dev/src/manifest.rs b/tools/dev/src/manifest.rs index 99d0a613c..7dbae7836 100644 --- a/tools/dev/src/manifest.rs +++ b/tools/dev/src/manifest.rs @@ -7,6 +7,7 @@ pub const LOOPBACK_HOST: &str = "127.0.0.1"; pub const ANY_HOST: &str = "0.0.0.0"; pub const DEV_PROXY_PORT: u16 = 8088; +pub const DEV_PROXY_GATEWAY_PORTS_ENV: &str = "FLUXER_DEV_PROXY_GATEWAY_PORTS"; pub const APP_PORT: u16 = 3000; pub const APP_PROXY_PORT: u16 = 8773; pub const ADMIN_PORT: u16 = 3020; diff --git a/tools/dev/src/media_proxy.rs b/tools/dev/src/media_proxy.rs index 414303011..e8dc3c50f 100644 --- a/tools/dev/src/media_proxy.rs +++ b/tools/dev/src/media_proxy.rs @@ -17,6 +17,8 @@ const DEV_S3_ACCESS_KEY_ID: &str = "fluxer"; const DEV_S3_SECRET_ACCESS_KEY: &str = "fluxer-secret"; const DEV_S3_HOST: &str = "127.0.0.1"; const DEV_S3_PORT: u16 = 8333; +const DEV_SEAWEEDFS_STOP_TIMEOUT: Duration = Duration::from_secs(15); +const DEV_SEAWEEDFS_KILL_TIMEOUT: Duration = Duration::from_secs(2); pub async fn run_dev_media_doctor( repair: bool, @@ -35,10 +37,20 @@ pub async fn run_dev_media_doctor( } pub async fn ensure_dev_object_store(repair: bool, repair_timeout_secs: u64) -> Result<()> { + let managed_process_running = + read_dev_seaweedfs_pid()?.is_some_and(managed_seaweedfs_process_running); + if repair && !tcp_reachable(DEV_S3_HOST, DEV_S3_PORT) && !managed_process_running { + println!("SeaweedFS S3 is not running; starting the managed development instance."); + start_dev_seaweedfs()?; + wait_s3_api(repair_timeout_secs).await?; + ensure_s3_buckets()?; + return Ok(()); + } + match wait_s3_api(5).await { Ok(()) => {} - Err(error) if repair => { - println!("SeaweedFS S3 check failed: {error}"); + Err(_) if repair => { + println!("SeaweedFS S3 is unresponsive; restarting the managed development instance."); start_dev_seaweedfs()?; wait_s3_api(repair_timeout_secs).await?; } @@ -128,7 +140,7 @@ fn start_dev_seaweedfs() -> Result<()> { if let Some(pid) = read_dev_seaweedfs_pid()? { if managed_seaweedfs_process_running(pid) { eprintln!("Stopping unresponsive managed SeaweedFS process {pid}."); - stop_managed_seaweedfs_process(pid); + stop_managed_seaweedfs_process(pid)?; } let _ = fs::remove_file(DEV_SEAWEEDFS_PID_FILE.as_path()); } @@ -141,15 +153,24 @@ fn start_dev_seaweedfs() -> Result<()> { let log_path = DEV_LOG_DIR.join("seaweedfs.log"); let log = OpenOptions::new() .create(true) - .append(true) + .write(true) + .truncate(true) .open(&log_path) .with_context(|| format!("failed to open SeaweedFS log {}", log_path.display()))?; let stderr = log .try_clone() .context("failed to clone SeaweedFS log handle")?; let data_dir_arg = format!("-dir={}", DEV_SEAWEEDFS_DIR.display()); - let child = Command::new(weed) - .args(["-logtostderr=true", "mini", &data_dir_arg]) + let mut child = Command::new(weed) + .args([ + "-logtostderr=true", + "mini", + &data_dir_arg, + "-admin.ui=false", + "-filer.disableDirListing", + "-s3.port.iceberg=0", + "-webdav=false", + ]) .env("AWS_ACCESS_KEY_ID", DEV_S3_ACCESS_KEY_ID) .env("AWS_SECRET_ACCESS_KEY", DEV_S3_SECRET_ACCESS_KEY) .env("S3_BUCKET", DEV_S3_BUCKETS) @@ -159,12 +180,16 @@ fn start_dev_seaweedfs() -> Result<()> { .spawn() .context("failed to start SeaweedFS")?; let pid = child.id(); - fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()).with_context(|| { - format!( - "failed to write SeaweedFS pid file {}", - DEV_SEAWEEDFS_PID_FILE.display() - ) - })?; + if let Err(error) = fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()) { + let _ = child.kill(); + let _ = child.wait(); + return Err(error).with_context(|| { + format!( + "failed to write SeaweedFS pid file {}", + DEV_SEAWEEDFS_PID_FILE.display() + ) + }); + } println!( "Started SeaweedFS dev object store with pid {pid}; logs: {}", log_path.display() @@ -195,20 +220,44 @@ fn managed_seaweedfs_process_running(pid: u32) -> bool { } #[cfg(unix)] -fn stop_managed_seaweedfs_process(pid: u32) { - unsafe { - libc::kill(pid as i32, libc::SIGTERM); - } - std::thread::sleep(Duration::from_secs(2)); - if managed_seaweedfs_process_running(pid) { - unsafe { - libc::kill(pid as i32, libc::SIGKILL); +fn stop_managed_seaweedfs_process(pid: u32) -> Result<()> { + let result = unsafe { libc::kill(pid as i32, libc::SIGTERM) }; + if result == -1 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + return Err(error).context("failed to terminate managed SeaweedFS process"); } } + + let terminate_deadline = Instant::now() + DEV_SEAWEEDFS_STOP_TIMEOUT; + while managed_seaweedfs_process_running(pid) && Instant::now() < terminate_deadline { + std::thread::sleep(Duration::from_millis(100)); + } + if !managed_seaweedfs_process_running(pid) { + return Ok(()); + } + + let result = unsafe { libc::kill(pid as i32, libc::SIGKILL) }; + if result == -1 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + return Err(error).context("failed to kill unresponsive managed SeaweedFS process"); + } + } + let kill_deadline = Instant::now() + DEV_SEAWEEDFS_KILL_TIMEOUT; + while managed_seaweedfs_process_running(pid) && Instant::now() < kill_deadline { + std::thread::sleep(Duration::from_millis(100)); + } + if managed_seaweedfs_process_running(pid) { + bail!("managed SeaweedFS process {pid} survived SIGKILL"); + } + Ok(()) } #[cfg(not(unix))] -fn stop_managed_seaweedfs_process(_pid: u32) {} +fn stop_managed_seaweedfs_process(_pid: u32) -> Result<()> { + Ok(()) +} async fn check_dev_media_path(base_url: &str, path: &str) -> Result<()> { let path = if path.starts_with('/') { diff --git a/tools/dev/src/object_store.rs b/tools/dev/src/object_store.rs index 4c9ee06a9..69bc4810e 100644 --- a/tools/dev/src/object_store.rs +++ b/tools/dev/src/object_store.rs @@ -168,11 +168,11 @@ fn s3_bucket_already_exists_output(output: &str) -> bool { output.contains("BucketAlreadyExists") || output.contains("BucketAlreadyOwnedByYou") } -pub async fn bootstrap_schema_and_object_store() -> Result<()> { +pub async fn bootstrap_schema() -> Result<()> { if cassandra_backend_enabled() { apply_schema(Some(config_from_env()?)).await?; } - ensure_s3_buckets() + Ok(()) } fn cassandra_backend_enabled() -> bool { diff --git a/tools/dev/src/proc.rs b/tools/dev/src/proc.rs index 95282721e..6c5b35d00 100644 --- a/tools/dev/src/proc.rs +++ b/tools/dev/src/proc.rs @@ -4,9 +4,11 @@ use crate::env::merge_default_env_with_current; use crate::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE}; use anyhow::{Context, Result, bail}; use std::collections::{BTreeMap, VecDeque}; +use std::future::Future; +use std::io::{BufRead, BufReader, Read}; use std::net::{TcpStream, ToSocketAddrs}; use std::path::Path; -use std::process::{Command, Output, Stdio}; +use std::process::{Child, Command, Output, Stdio}; use std::time::{Duration, Instant}; use tokio::time::sleep; @@ -172,6 +174,152 @@ pub fn run_command(args: &[&str], options: RunOptions<'_>) -> Result { Ok(output) } +pub async fn run_command_interruptible( + args: &[&str], + options: RunOptions<'_>, + shutdown: &mut ShutdownSignal, +) -> Result> { + if options.capture { + bail!("interruptible commands do not support captured output"); + } + println!("$ {}", format_command(args)); + let env = merged_env(Some(&options.env), options.load_default_env)?; + let mut command = Command::new(args[0]); + command + .args(&args[1..]) + .current_dir(options.cwd) + .env_clear() + .envs(env) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + configure_process_group(&mut command); + let mut child = command + .spawn() + .with_context(|| format!("failed to run {}", format_command(args)))?; + loop { + let status = match child.try_wait() { + Ok(status) => status, + Err(error) => { + stop_process_group(&mut child, Duration::from_secs(5)).await; + return Err(error) + .with_context(|| format!("failed to wait for {}", format_command(args))); + } + }; + if let Some(status) = status { + stop_process_group(&mut child, Duration::from_secs(5)).await; + let output = Output { + status, + stdout: Vec::new(), + stderr: Vec::new(), + }; + if options.check && !output.status.success() { + let code = output.status.code().unwrap_or(-1); + bail!( + "Command failed with exit code {code}: {}", + format_command(args) + ); + } + return Ok(AwaitOutcome::Completed(output)); + } + tokio::select! { + signal = shutdown.recv() => { + stop_process_group(&mut child, Duration::from_secs(5)).await; + return Ok(AwaitOutcome::Shutdown(signal)); + } + _ = sleep(Duration::from_millis(100)) => {} + } + } +} + +pub fn configure_process_group(command: &mut Command) { + #[cfg(unix)] + unsafe { + use std::os::unix::process::CommandExt; + command.pre_exec(|| { + if libc::setsid() == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } +} + +#[cfg(unix)] +pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> { + signal_process_group(process, libc::SIGTERM) +} + +#[cfg(not(unix))] +pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> { + process.kill() +} + +#[cfg(unix)] +pub fn force_kill_process_group(process: &mut Child) { + if let Err(error) = signal_process_group(process, libc::SIGKILL) { + eprintln!( + "Failed to send SIGKILL to process group {}: {error}", + process.id() + ); + } + let _ = process.kill(); + let _ = process.wait(); +} + +#[cfg(unix)] +pub fn process_group_running(process: &mut Child) -> bool { + if unsafe { libc::kill(-(process.id() as i32), 0) } == 0 { + return true; + } + let error = std::io::Error::last_os_error(); + error.raw_os_error() != Some(libc::ESRCH) +} + +#[cfg(not(unix))] +pub fn process_group_running(process: &mut Child) -> bool { + process.try_wait().ok().flatten().is_none() +} + +#[cfg(not(unix))] +pub fn force_kill_process_group(process: &mut Child) { + let _ = process.kill(); + let _ = process.wait(); +} + +#[cfg(unix)] +fn signal_process_group(process: &Child, signal: libc::c_int) -> std::io::Result<()> { + if unsafe { libc::kill(-(process.id() as i32), signal) } == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + return Ok(()); + } + Err(error) +} + +pub async fn stop_process_group(process: &mut Child, grace_period: Duration) { + if let Err(error) = terminate_process_group(process) { + eprintln!( + "Failed to terminate process group {}: {error}", + process.id() + ); + } + let deadline = Instant::now() + grace_period; + loop { + let leader_exited = process.try_wait().ok().flatten().is_some(); + if leader_exited && !process_group_running(process) { + return; + } + if Instant::now() >= deadline { + force_kill_process_group(process); + return; + } + sleep(Duration::from_millis(100)).await; + } +} + pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> Result<()> { let deadline = Instant::now() + Duration::from_secs(timeout_secs); let mut last_error = None; @@ -199,6 +347,19 @@ pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> R } pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> { + wait_http_status(name, url, timeout_secs, |status| status.as_u16() < 500).await +} + +pub async fn wait_http_success(name: &str, url: &str, timeout_secs: u64) -> Result<()> { + wait_http_status(name, url, timeout_secs, |status| status.is_success()).await +} + +async fn wait_http_status( + name: &str, + url: &str, + timeout_secs: u64, + accepts: impl Fn(reqwest::StatusCode) -> bool, +) -> Result<()> { let client = reqwest::Client::builder() .timeout(Duration::from_secs(5)) .build()?; @@ -206,7 +367,7 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> { let mut last_error = None; while Instant::now() < deadline { match client.get(url).send().await { - Ok(response) if response.status().as_u16() < 500 => { + Ok(response) if accepts(response.status()) => { println!("{name} is reachable at {url}"); return Ok(()); } @@ -221,6 +382,30 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> { ); } +pub fn prefix_output(label: &str, reader: impl Read) { + let mut reader = BufReader::new(reader); + let mut line = Vec::new(); + loop { + line.clear(); + match reader.read_until(b'\n', &mut line) { + Ok(0) => return, + Ok(_) => { + if line.last() == Some(&b'\n') { + line.pop(); + } + if line.last() == Some(&b'\r') { + line.pop(); + } + println!("[{label}] {}", String::from_utf8_lossy(&line)); + } + Err(error) => { + eprintln!("[{label}] output reader failed: {error}"); + return; + } + } + } +} + pub const RESTART_WINDOW: Duration = Duration::from_secs(60); pub const RESTART_LIMIT: usize = 5; @@ -262,6 +447,29 @@ impl ShutdownSignal { } } +pub enum AwaitOutcome { + Completed(T), + Shutdown(&'static str), +} + +pub async fn await_or_shutdown(shutdown: &mut ShutdownSignal, future: F) -> AwaitOutcome +where + F: Future, +{ + tokio::select! { + biased; + signal = shutdown.recv() => AwaitOutcome::Shutdown(signal), + output = future => AwaitOutcome::Completed(output), + } +} + +pub async fn pending_shutdown(shutdown: &mut ShutdownSignal) -> Option<&'static str> { + match await_or_shutdown(shutdown, std::future::ready(())).await { + AwaitOutcome::Completed(()) => None, + AwaitOutcome::Shutdown(signal) => Some(signal), + } +} + #[cfg(not(unix))] pub struct ShutdownSignal; diff --git a/tools/dev/src/proxy.rs b/tools/dev/src/proxy.rs index 0920f4c11..5f88001ee 100644 --- a/tools/dev/src/proxy.rs +++ b/tools/dev/src/proxy.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use crate::manifest::{ANY_HOST, DEV_PROXY_PORT, LOCAL_APP_URL, PROXY_ROUTES, ProxyRoute}; +use crate::manifest::{ + ANY_HOST, DEV_PROXY_GATEWAY_PORTS_ENV, DEV_PROXY_PORT, GATEWAY_PORT, LOCAL_APP_URL, + PROXY_ROUTES, ProxyRoute, +}; use anyhow::{Context, Result, bail}; use axum::{ Router, @@ -14,7 +17,7 @@ use hyper_util::rt::TokioIo; use std::collections::HashMap; use std::env; use std::net::SocketAddr; -use std::sync::{LazyLock, Mutex}; +use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; @@ -26,6 +29,7 @@ static ROUTE_CURSORS: LazyLock>> = #[derive(Clone)] struct ProxyState { http_client: reqwest::Client, + gateway_ports: Arc<[u16]>, } const BLOCKED_REQUEST_HEADERS: &[&str] = &[ @@ -67,9 +71,21 @@ pub async fn run_proxy(host: &str, port: u16) -> Result<()> { .tcp_nodelay(true) .build() .context("failed to build dev proxy HTTP client")?; + let gateway_ports = gateway_proxy_ports()?; + println!( + "Fluxer dev proxy gateway ports: {}", + gateway_ports + .iter() + .map(u16::to_string) + .collect::>() + .join(",") + ); let app = Router::new() .fallback(any(proxy_request)) - .with_state(ProxyState { http_client }); + .with_state(ProxyState { + http_client, + gateway_ports, + }); axum::serve( listener, @@ -91,11 +107,29 @@ async fn proxy_request( } let route = route_for_path(&request_head.path); + let (target_host, target_port) = target_for_route(route, &state.gateway_ports); if is_upgrade_request(&request_head) { - return proxy_upgrade(request, request_head, route, client_addr).await; + return proxy_upgrade( + request, + request_head, + route, + target_host, + target_port, + client_addr, + ) + .await; } - proxy_http(state, request, request_head, route, client_addr).await + proxy_http( + state, + request, + request_head, + route, + target_host, + target_port, + client_addr, + ) + .await } async fn proxy_http( @@ -103,9 +137,10 @@ async fn proxy_http( request: Request, request_head: RequestHead, route: &'static ProxyRoute, + target_host: &'static str, + target_port: u16, client_addr: SocketAddr, ) -> Response { - let (target_host, target_port) = target_for_route(route); let target_url = upstream_http_url(&request_head.path, route, target_host, target_port); let (parts, body) = request.into_parts(); let method = parts.method.clone(); @@ -155,10 +190,11 @@ async fn proxy_upgrade( mut request: Request, request_head: RequestHead, route: &'static ProxyRoute, + target_host: &'static str, + target_port: u16, client_addr: SocketAddr, ) -> Response { let on_upgrade = hyper::upgrade::on(&mut request); - let (target_host, target_port) = target_for_route(route); let mut target = match TcpStream::connect((target_host, target_port)).await { Ok(target) => target, Err(error) => return bad_gateway_response(error), @@ -497,17 +533,59 @@ pub fn route_for_path(path: &str) -> &'static ProxyRoute { PROXY_ROUTES.last().expect("proxy has fallback route") } -pub fn target_for_route(route: &'static ProxyRoute) -> (&'static str, u16) { - if route.alternate_ports.is_empty() { +pub fn target_for_route(route: &'static ProxyRoute, gateway_ports: &[u16]) -> (&'static str, u16) { + let uses_gateway_ports = route.prefix == "/gateway"; + let port_count = if uses_gateway_ports { + gateway_ports.len() + } else { + route.alternate_ports.len() + 1 + }; + assert!( + port_count > 0, + "proxy route must have at least one target port" + ); + if port_count == 1 { + if uses_gateway_ports { + return (route.host, gateway_ports[0]); + } return (route.host, route.port); } - let ports = std::iter::once(route.port) - .chain(route.alternate_ports.iter().copied()) - .collect::>(); let mut cursors = ROUTE_CURSORS.lock().expect("route cursor lock poisoned"); let cursor = *cursors.get(route.prefix).unwrap_or(&0); - cursors.insert(route.prefix, (cursor + 1) % ports.len()); - (route.host, ports[cursor]) + cursors.insert(route.prefix, (cursor + 1) % port_count); + let port = if uses_gateway_ports { + gateway_ports[cursor] + } else if cursor == 0 { + route.port + } else { + route.alternate_ports[cursor - 1] + }; + (route.host, port) +} + +fn gateway_proxy_ports() -> Result> { + let raw = env::var(DEV_PROXY_GATEWAY_PORTS_ENV).unwrap_or_else(|_| GATEWAY_PORT.to_string()); + let mut ports = Vec::new(); + for token in raw.split(',') { + let token = token.trim(); + if token.is_empty() { + bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains an empty port"); + } + let port = token + .parse::() + .with_context(|| format!("Invalid port {token:?} in {DEV_PROXY_GATEWAY_PORTS_ENV}"))?; + if port == 0 { + bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} ports must be greater than zero"); + } + if ports.contains(&port) { + bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains duplicate port {port}"); + } + ports.push(port); + } + if ports.len() > 3 { + bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} supports at most three ports"); + } + Ok(ports.into()) } pub fn rewrite_path(path: &str, route: &ProxyRoute) -> String { diff --git a/tools/dev/src/rust_services.rs b/tools/dev/src/rust_services.rs index e32bcc00d..33b1c3099 100644 --- a/tools/dev/src/rust_services.rs +++ b/tools/dev/src/rust_services.rs @@ -1,10 +1,11 @@ // SPDX-License-Identifier: AGPL-3.0-or-later use crate::manifest::{DEV_PROXY_PORT, MEDIA_PROXY_PORT, RustServiceSpec, rust_services}; -use crate::paths::ROOT; +use crate::paths::{ROOT, TARGET_DIR}; use crate::proc::{ - RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env, - restart_budget_exceeded, run_command, + AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown, + configure_process_group, format_command, merged_env, pending_shutdown, prefix_output, + restart_budget_exceeded, run_command_interruptible, }; use anyhow::{Result, bail}; use std::collections::{BTreeSet, VecDeque}; @@ -12,6 +13,8 @@ use std::env; #[cfg(target_os = "linux")] use std::fs; use std::net::{SocketAddr, TcpListener}; +#[cfg(target_os = "linux")] +use std::path::Path; use std::process::{Child, Command, Stdio}; use std::time::{Duration, Instant}; use tokio::time::sleep; @@ -34,26 +37,51 @@ impl SupervisedService { pub async fn run_rust_services(service_names: &[String]) -> Result { let selected = select_services(service_names)?; - cleanup_orphaned_service_processes(&selected).await?; - wait_for_service_ports_available(&selected)?; - build_services(&selected)?; let mut shutdown = ShutdownSignal::new()?; + cleanup_orphaned_service_processes(&selected).await?; + if let Some(signal) = pending_shutdown(&mut shutdown).await { + println!("Received {signal}; stopping Rust service startup..."); + return Ok(0); + } + wait_for_service_ports_available(&selected)?; + match build_services(&selected, &mut shutdown).await? { + AwaitOutcome::Completed(_) => {} + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping Rust service startup..."); + return Ok(0); + } + } let mut supervised = Vec::new(); for spec in &selected { for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] { + let child = match start_service(spec, mode, port) { + Ok(child) => child, + Err(error) => { + stop_supervised_services(&mut supervised); + return Err(error); + } + }; supervised.push(SupervisedService { spec: spec.clone(), mode, port, - child: start_service(spec, mode, port)?, + child, restarts: VecDeque::new(), }); } } loop { - if let Err(error) = restart_exited_services(&mut supervised).await { - stop_supervised_services(&mut supervised); - return Err(error); + match await_or_shutdown(&mut shutdown, restart_exited_services(&mut supervised)).await { + AwaitOutcome::Completed(Ok(())) => {} + AwaitOutcome::Completed(Err(error)) => { + stop_supervised_services(&mut supervised); + return Err(error); + } + AwaitOutcome::Shutdown(signal) => { + println!("Received {signal}; stopping Rust service tasks..."); + stop_supervised_services(&mut supervised); + return Ok(0); + } } tokio::select! { signal = shutdown.recv() => { @@ -71,6 +99,7 @@ async fn restart_exited_services(supervised: &mut [SupervisedService]) -> Result let Some(status) = entry.child.try_wait()? else { continue; }; + crate::gateway::stop_child_processes(&mut [&mut entry.child]); if restart_budget_exceeded(&mut entry.restarts, Instant::now()) { bail!( "Rust service {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up", @@ -145,7 +174,10 @@ pub fn select_services(service_names: &[String]) -> Result> .collect()) } -fn build_services(services: &[RustServiceSpec]) -> Result<()> { +async fn build_services( + services: &[RustServiceSpec], + shutdown: &mut ShutdownSignal, +) -> Result> { let mut packages = Vec::new(); let mut seen = BTreeSet::new(); for spec in services { @@ -156,7 +188,23 @@ fn build_services(services: &[RustServiceSpec]) -> Result<()> { let mut args = vec!["cargo".to_owned(), "build".to_owned()]; args.extend(packages); let refs = args.iter().map(String::as_str).collect::>(); - run_command(&refs, RunOptions::default()).map(drop) + let env = if env::var_os("CARGO_BUILD_JOBS").is_none() { + vec![( + "CARGO_BUILD_JOBS".to_owned(), + Some(env::var("FLUXER_DEV_CARGO_JOBS").unwrap_or_else(|_| "2".to_owned())), + )] + } else { + Vec::new() + }; + run_command_interruptible( + &refs, + RunOptions { + env, + ..RunOptions::default() + }, + shutdown, + ) + .await } fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result { @@ -172,14 +220,7 @@ fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result .envs(env) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - #[cfg(unix)] - unsafe { - use std::os::unix::process::CommandExt; - command.pre_exec(|| { - libc::setsid(); - Ok(()) - }); - } + configure_process_group(&mut command); let mut child = command.spawn()?; if let Some(stdout) = child.stdout.take() { let stdout_label = label.clone(); @@ -212,11 +253,20 @@ pub fn service_command(spec: &RustServiceSpec) -> Vec { format!("run -p {}", spec.package), ]; } + let target_dir = env::var_os("CARGO_TARGET_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| TARGET_DIR.clone()); + let target_dir = if target_dir.is_absolute() { + target_dir + } else { + ROOT.join(target_dir) + }; vec![ - "cargo".to_owned(), - "run".to_owned(), - "-p".to_owned(), - spec.package.to_owned(), + target_dir + .join("debug") + .join(format!("{}{}", spec.package, std::env::consts::EXE_SUFFIX)) + .display() + .to_string(), ] } @@ -238,18 +288,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String .unwrap_or_else(|_| "nats://nats:4222".to_owned()), ), ), - ( - "FLUXER_CASSANDRA_HOSTS".to_owned(), - Some(env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned())), - ), - ( - "FLUXER_CASSANDRA_KEYSPACE".to_owned(), - Some(env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned())), - ), - ( - "FLUXER_CASSANDRA_PORT".to_owned(), - Some(env::var("FLUXER_CASSANDRA_PORT").unwrap_or_else(|_| "9042".to_owned())), - ), ]; if mode == "shard" { envs.push(("FLUXER_SVC_SHARD_ID".to_owned(), Some("0".to_owned()))); @@ -284,13 +322,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String envs } -fn prefix_output(label: &str, reader: impl std::io::Read) { - use std::io::{BufRead, BufReader}; - for line in BufReader::new(reader).lines().map_while(|line| line.ok()) { - println!("[{label}] {line}"); - } -} - fn wait_for_service_ports_available(services: &[RustServiceSpec]) -> Result<()> { let conflicts = services .iter() @@ -333,7 +364,7 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Res "Stopping orphaned Rust service process group(s): {}", format_pids(&pids) ); - let term_failed = signal_process_groups(&pids, libc::SIGTERM); + let term_failed = signal_process_groups(&leaders, libc::SIGTERM); if !term_failed.is_empty() { println!( "SIGTERM delivery failed for orphaned Rust service pid(s): {}", @@ -348,7 +379,12 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Res return Ok(()); } if Instant::now() >= deadline { - let kill_failed = signal_process_groups(&remaining, libc::SIGKILL); + let remaining_leaders = leaders + .iter() + .filter(|leader| remaining.contains(&leader.pid)) + .cloned() + .collect::>(); + let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL); sleep(Duration::from_millis(200)).await; let survivors = surviving_service_group_pids(&leaders)?; if survivors.is_empty() { @@ -380,19 +416,85 @@ async fn cleanup_orphaned_service_processes(_services: &[RustServiceSpec]) -> Re } #[cfg(target_os = "linux")] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] struct RustServiceLeader { + pid: i32, + members: Vec, +} + +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct RustServiceProcessIdentity { pid: i32, starttime: u64, } +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct RustServiceProcess { + pid: i32, + ppid: i32, + pgid: i32, + state: char, + starttime: u64, +} + #[cfg(target_os = "linux")] fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result> { - let binaries = services + let current_exe = std::env::current_exe()?; + let processes = service_process_snapshot()?; + let mut owned_pids = BTreeSet::new(); + for process in &processes { + if process.ppid != 1 || proc_stat_state_is_dead(process.state) { + continue; + } + let args = proc_cmdline(process.pid); + let orphaned_supervisor = cmdline_is_service_supervisor(&args, ¤t_exe); + let orphaned_service = proc_has_managed_service_environment(process.pid, services) + && fs::read_link(format!("/proc/{}/cwd", process.pid)) + .ok() + .as_deref() + == Some(ROOT.as_path()); + if orphaned_supervisor || orphaned_service { + owned_pids.insert(process.pid); + } + } + loop { + let previous_count = owned_pids.len(); + for process in &processes { + if owned_pids.contains(&process.ppid) { + owned_pids.insert(process.pid); + } + } + if owned_pids.len() == previous_count { + break; + } + } + let group_pids = processes .iter() - .map(|spec| format!("target/debug/{}", spec.package)) + .filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1) + .map(|process| process.pgid) .collect::>(); - let mut leaders = Vec::new(); + let leaders = group_pids + .into_iter() + .map(|pid| RustServiceLeader { + pid, + members: processes + .iter() + .filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid) + .map(|process| RustServiceProcessIdentity { + pid: process.pid, + starttime: process.starttime, + }) + .collect(), + }) + .collect::>(); + Ok(leaders) +} + +#[cfg(target_os = "linux")] +fn service_process_snapshot() -> Result> { + let mut processes = Vec::new(); for entry in fs::read_dir("/proc")? { let Ok(entry) = entry else { continue; @@ -404,18 +506,69 @@ fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result bool { + args.first().map(Path::new) == Some(current_exe) + && args.get(1).map(String::as_str) == Some("rust-services") +} + +#[cfg(target_os = "linux")] +fn proc_has_managed_service_environment(pid: i32, services: &[RustServiceSpec]) -> bool { + let environment_bytes = fs::read(format!("/proc/{pid}/environ")).unwrap_or_default(); + let environment = environment_bytes + .split(|byte| *byte == 0) + .filter_map(|entry| { + let separator = entry.iter().position(|byte| *byte == b'=')?; + let (key, value) = entry.split_at(separator); + Some((key, &value[1..])) + }) + .filter_map(|(key, value)| { + Some(( + std::str::from_utf8(key).ok()?, + std::str::from_utf8(value).ok()?, + )) + }) + .collect::>(); + let Some(name) = environment.get("FLUXER_SVC_NAME") else { + return false; + }; + let Some(mode) = environment.get("FLUXER_SVC_MODE") else { + return false; + }; + let Some(port) = environment + .get("FLUXER_SVC_PORT") + .and_then(|port| port.parse::().ok()) + else { + return false; + }; + if environment.get("FLUXER_SVC_LISTEN_HOST") != Some(&"0.0.0.0") { + return false; + } + services.iter().any(|service| { + service.name == *name + && match *mode { + "router" => port == service.port_base, + "shard" => port == service.port_base + 1, + _ => false, + } + }) } #[cfg(target_os = "linux")] @@ -429,7 +582,7 @@ fn format_pids(pids: &[i32]) -> String { .join(", ") } -#[cfg(any(target_os = "linux", test))] +#[cfg(test)] fn cmdline_has_service_binary(args: &[String], binaries: &BTreeSet) -> bool { args.first() .map(|arg| binaries.iter().any(|binary| arg.ends_with(binary))) @@ -446,30 +599,6 @@ fn proc_cmdline(pid: i32) -> Vec { .collect() } -#[cfg(target_os = "linux")] -fn proc_parent_pid(pid: i32) -> Option { - fs::read_to_string(format!("/proc/{pid}/status")) - .ok()? - .lines() - .find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok()) -} - -#[cfg(target_os = "linux")] -fn process_exists(pid: i32) -> bool { - assert!(pid > 0); - match proc_stat_state_and_pgid(pid) { - Some((state, _pgid)) => !proc_stat_state_is_dead(state), - None => false, - } -} - -#[cfg(target_os = "linux")] -fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> { - assert!(pid > 0); - let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; - parse_proc_stat_state_and_pgid(&stat) -} - #[cfg(any(target_os = "linux", test))] fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> { let (_, after_comm) = stat.rsplit_once(')')?; @@ -481,10 +610,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> { } #[cfg(target_os = "linux")] -fn proc_stat_starttime(pid: i32) -> Option { - assert!(pid > 0); - let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; - parse_proc_stat_starttime(&stat) +fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> { + let (_, after_comm) = stat.rsplit_once(')')?; + let mut fields = after_comm.split_ascii_whitespace(); + let state = fields.next()?.chars().next()?; + let ppid = fields.next()?.parse().ok()?; + let pgid = fields.next()?.parse().ok()?; + Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?)) } #[cfg(any(target_os = "linux", test))] @@ -500,62 +632,43 @@ fn proc_stat_state_is_dead(state: char) -> bool { #[cfg(target_os = "linux")] fn surviving_service_group_pids(leaders: &[RustServiceLeader]) -> Result> { - let leader_pids = leaders + Ok(leaders .iter() + .filter(|leader| service_group_has_owned_member(leader)) .map(|leader| leader.pid) - .collect::>(); - let mut survivors = Vec::new(); - for entry in fs::read_dir("/proc")? { - let Ok(entry) = entry else { - continue; - }; - let Some(pid) = entry - .file_name() - .to_str() - .and_then(|name| name.parse::().ok()) - else { - continue; - }; - let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { - continue; - }; - let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else { - continue; - }; - if proc_stat_state_is_dead(state) { - continue; - } - if leader_pids.contains(&pgid) { - survivors.push(pid); - continue; - } - let leader = leaders.iter().find(|leader| leader.pid == pid); - if let Some(leader) = leader - && parse_proc_stat_starttime(&stat) == Some(leader.starttime) - { - survivors.push(pid); - } - } - survivors.sort_unstable(); - Ok(survivors) + .collect()) } #[cfg(target_os = "linux")] -fn signal_process_groups(pids: &[i32], signal: i32) -> Vec { +fn service_group_has_owned_member(leader: &RustServiceLeader) -> bool { + leader.members.iter().any(|member| { + let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else { + return false; + }; + let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else { + return false; + }; + !proc_stat_state_is_dead(state) + && pgid == leader.pid + && parse_proc_stat_starttime(&stat) == Some(member.starttime) + }) +} + +#[cfg(target_os = "linux")] +fn signal_process_groups(leaders: &[RustServiceLeader], signal: i32) -> Vec { assert!(signal == libc::SIGTERM || signal == libc::SIGKILL); - let mut failed = Vec::with_capacity(pids.len()); - for pid in pids { - assert!(*pid > 0); - let group_result = unsafe { libc::kill(-pid, signal) }; + let mut failed = Vec::with_capacity(leaders.len()); + for leader in leaders { + assert!(leader.pid > 1); + if !service_group_has_owned_member(leader) { + continue; + } + let group_result = unsafe { libc::kill(-leader.pid, signal) }; if group_result == 0 { continue; } - let direct_result = unsafe { libc::kill(*pid, signal) }; - if direct_result == 0 { - continue; - } - if process_exists(*pid) { - failed.push(*pid); + if service_group_has_owned_member(leader) { + failed.push(leader.pid); } } failed diff --git a/tools/dev/src/tunnel.rs b/tools/dev/src/tunnel.rs index 6b46840cd..22558f3ed 100644 --- a/tools/dev/src/tunnel.rs +++ b/tools/dev/src/tunnel.rs @@ -253,6 +253,16 @@ pub async fn run_cloudflare_tunnel( return Ok(status.code().unwrap_or(1)); } if running_inside_devcontainer() { + let container = std::fs::read_to_string("/etc/hostname") + .context("failed to read the current devcontainer hostname")?; + let container = container.trim(); + if container.is_empty() + || !container + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-')) + { + bail!("Current devcontainer hostname is invalid"); + } let docker = docker_command(); let mut command = Command::new(&docker[0]); command.args(&docker[1..]); @@ -261,11 +271,7 @@ pub async fn run_cloudflare_tunnel( "run", "--rm", "--network", - &format!( - "container:{}", - std::env::var("HOSTNAME") - .unwrap_or_else(|_| "fluxer-dev-workspace-1".to_owned()) - ), + &format!("container:{container}"), "cloudflare/cloudflared:latest", "tunnel", "run",