fluxerapp/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer.git synced 2026-09-02 21:04:06 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(workspace): stabilise the development stack (#2024)

Browse Source
This commit is contained in:
Hampus
2026-08-26 18:52:53 +02:00
committed by GitHub
parent cf1a7d7a8a
commit 5d2e5932a4
55 changed files with 1848 additions and 3396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.devcontainer/Dockerfile
+10 -32
View File
@@ -7,7 +7,6 @@ ARG USER_UID=1000
ARG USER_GID=1000
ARG NODE_MAJOR=24
ARG ELP_VERSION=2026-02-27
ARG HELM_VERSION=4.2.0
ARG PNPM_VERSION=10.29.3
ARG WASM_BINDGEN_VERSION=0.2.122
@@ -15,8 +14,8 @@ ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
acl \
bash \
brotli \
build-essential \
ca-certificates \
clang \
@@ -42,13 +41,13 @@ RUN apt-get update \
libfido2-dev \
libgbm1 \
libgtk-3-0 \
libimage-exiftool-perl \
libnotify4 \
libnss3 \
libpipewire-0.3-dev \
libpulse-dev \
libsecret-1-0 \
libudev-dev \
libuv1-dev \
libcurl4-openssl-dev \
libswresample-dev \
libswscale-dev \
@@ -71,14 +70,12 @@ RUN apt-get update \
ninja-build \
openssh-client \
pkg-config \
protobuf-compiler \
python3 \
python3-pip \
rsync \
python3-venv \
sudo \
rpm \
unzip \
webp \
xz-utils \
xdg-utils \
zstd \
@@ -90,6 +87,7 @@ RUN apt-get update \
bat \
btop \
docker-cli \
docker-compose \
dnsutils \
fd-find \
gdb \
@@ -122,24 +120,12 @@ RUN apt-get update \
&& ln -sf /usr/bin/batcat /usr/local/bin/bat \
&& rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \
RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \
&& apt-get install -y --no-install-recommends nodejs \
&& rm -rf /var/lib/apt/lists/* \
&& corepack enable
RUN ARCH="$(dpkg --print-architecture)" \
&& case "$ARCH" in \
amd64) HELM_ARCH="amd64" ;; \
arm64) HELM_ARCH="arm64" ;; \
*) echo "Unsupported architecture for Helm: $ARCH" >&2; exit 1 ;; \
esac \
&& curl -fsSL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${HELM_ARCH}.tar.gz" -o /tmp/helm.tgz \
&& tar -C /tmp -xzf /tmp/helm.tgz "linux-${HELM_ARCH}/helm" \
&& mv "/tmp/linux-${HELM_ARCH}/helm" /usr/local/bin/helm \
&& chmod +x /usr/local/bin/helm \
&& rm -rf /tmp/helm.tgz "/tmp/linux-${HELM_ARCH}"
RUN python3 -m pip install --break-system-packages --no-cache-dir awscli cqlsh
RUN python3 -m pip install --break-system-packages --no-cache-dir awscli
COPY tools/fonts/requirements.txt /tmp/fluxer-fonts-requirements.txt
RUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer-fonts-requirements.txt \
@@ -147,18 +133,13 @@ RUN python3 -m pip install --break-system-packages --no-cache-dir -r /tmp/fluxer
&& pyftsubset --help >/dev/null \
&& python3 -c "import fontTools, brotli"
RUN if ! command -v rebar3 >/dev/null 2>&1; then \
curl -fsSL https://s3.amazonaws.com/rebar3/rebar3 -o /usr/local/bin/rebar3 \
&& chmod +x /usr/local/bin/rebar3; \
fi
RUN ARCH="$(dpkg --print-architecture)" \
&& case "$ARCH" in \
amd64) ELP_ARCH="x86_64" ;; \
arm64) ELP_ARCH="aarch64" ;; \
*) echo "Unsupported architecture for ELP: $ARCH" >&2; exit 1 ;; \
esac \
&& curl -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \
&& curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL "https://github.com/WhatsApp/erlang-language-platform/releases/download/${ELP_VERSION}/elp-linux-${ELP_ARCH}-unknown-linux-gnu-otp-28.tar.gz" -o /tmp/elp.tgz \
&& tar -C /usr/local/bin -xzf /tmp/elp.tgz elp \
&& chmod +x /usr/local/bin/elp \
&& rm /tmp/elp.tgz
@@ -179,16 +160,13 @@ ENV DOCKER_HOST="unix:///var/run/docker.sock" \
ENV CC_wasm32_unknown_unknown="clang" \
AR_wasm32_unknown_unknown="llvm-ar"
RUN curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile default --component clippy,rustfmt \
RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --component clippy,rustfmt \
&& rustup target add wasm32-unknown-unknown \
&& cargo install cargo-watch --locked \
&& cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked
&& cargo install wasm-bindgen-cli --version "${WASM_BINDGEN_VERSION}" --locked \
&& rm -rf "/home/${USERNAME}/.cargo/registry" "/home/${USERNAME}/.cargo/git"
RUN corepack prepare "pnpm@${PNPM_VERSION}" --activate \
&& pnpm --version
RUN sudo apt-get update \
&& sudo apt-get install -y --no-install-recommends python3-venv \
&& sudo rm -rf /var/lib/apt/lists/*
WORKDIR /workspaces/fluxer
.devcontainer/devcontainer.json
+11 -35
View File
@@ -5,20 +5,17 @@
"workspaceFolder": "/workspaces/fluxer",
"shutdownAction": "stopCompose",
"remoteUser": "vscode",
"hostRequirements": {
"cpus": 4,
"memory": "8gb",
"storage": "32gb"
},
"remoteEnv": {
"DOCKER_HOST": "unix:///var/run/docker.sock"
},
"runServices": ["workspace", "postgres", "valkey", "nats", "livekit", "meilisearch", "mailpit"],
"forwardPorts": [
3000, 8088, 8080, 8771, 8082, 3010, 3020, 8100, 8101, 8102, 8103, 8104, 8105, 8106, 8107, 8108, 8109, 8110, 8111,
8112, 8113, 8114, 8115, 8116, 8117, 8118, 8119, 8120, 8121, 8122, 8123, 8124, 8125, 3900, 8888, 9333, 9340, 23646,
4222, 7700, 7880, 7900, 9200, 8000
],
"forwardPorts": [3000, 8088, 8080, 8771, 8082, 8773, 3010, 3020, 8333],
"portsAttributes": {
"8000": {
"label": "Zensical docs",
"onAutoForward": "openBrowserOnce"
},
"8088": {
"label": "Fluxer dev proxy",
"onAutoForward": "notify"
@@ -29,36 +26,15 @@
"3020": {
"label": "Fluxer admin"
},
"8100": {
"label": "Fluxer Rust service health"
},
"3900": {
"8333": {
"label": "SeaweedFS S3"
},
"8888": {
"label": "SeaweedFS filer"
},
"9333": {
"label": "SeaweedFS master"
},
"9340": {
"label": "SeaweedFS volume"
},
"23646": {
"label": "SeaweedFS admin"
},
"7880": {
"label": "LiveKit"
},
"7700": {
"label": "Meilisearch"
},
"9200": {
"label": "Elasticsearch"
"8773": {
"label": "Fluxer app proxy"
}
},
"postCreateCommand": "sudo chown -R vscode:vscode /workspaces/fluxer/target && find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -exec sudo chown -R vscode:vscode {} + && sudo chown -R vscode:vscode /home/vscode/.local/share/pnpm && cargo run -p fluxer-dev -- bootstrap",
"postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && cargo run -p fluxer-dev -- post-start && bash /workspaces/fluxer/fluxer_docs/serve.sh --daemon",
"postCreateCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- bootstrap",
"postStartCommand": "bash /workspaces/fluxer/.devcontainer/fix-docker-socket.sh && bash /workspaces/fluxer/.devcontainer/fix-workspace-permissions.sh && cargo run -p fluxer-dev -- post-start",
"customizations": {
"vscode": {
"settings": {
.devcontainer/docker-compose.yml
+68 -69
View File
@@ -1,5 +1,3 @@
name: fluxer-dev
services:
workspace:
build:
@@ -7,15 +5,29 @@ services:
dockerfile: .devcontainer/Dockerfile
command: sleep infinity
init: true
env_file:
- ../config/env/development.env
environment:
FLUXER_SEARCH_ENGINE: meilisearch
FLUXER_SEARCH_URL: http://meilisearch:7700
FLUXER_SEARCH_API_KEY: fluxer-dev-meilisearch
FLUXER_POSTGRES_HOST: postgres
FLUXER_SELF_HOSTED: "true"
DOCKER_HOST: unix:///var/run/docker.sock
npm_config_store_dir: /home/vscode/.local/share/pnpm/store
FLUXER_PUBLIC_PORT: "${FLUXER_DEV_PROXY_PORT:-8088}"
FLUXER_PUBLIC_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
FLUXER_API_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"
FLUXER_API_CLIENT_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"
FLUXER_APP_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
FLUXER_GATEWAY_ENDPOINT: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/gateway"
FLUXER_MEDIA_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
FLUXER_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
FLUXER_ADMIN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin"
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
FLUXER_S3_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
FLUXER_LIVEKIT_URL: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/livekit"
FLUXER_LIVEKIT_INTERNAL_URL: "http://livekit:7880"
FLUXER_LIVEKIT_WEBHOOK_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api/webhooks/livekit"
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
FLUXER_ADMIN_OAUTH_REDIRECT_URI: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/admin/oauth2_callback"
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: "http://localhost,http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api"
volumes:
- ..:/workspaces/fluxer:cached
- type: volume
@@ -229,6 +241,7 @@ services:
volume:
nocopy: true
- pnpm-store:/home/vscode/.local/share/pnpm/store
- docs-venv:/workspaces/fluxer/fluxer_docs/.venv
- cargo-registry:/home/vscode/.cargo/registry
- cargo-git:/home/vscode/.cargo/git
- rust-target:/workspaces/fluxer/target
@@ -236,45 +249,31 @@ services:
source: ${FLUXER_DOCKER_SOCKET:-/var/run/docker.sock}
target: /var/run/docker.sock
ports:
- "${FLUXER_DEV_DOCS_PORT:-8000}:8000"
- "${FLUXER_DEV_RSPACK_PORT:-3000}:3000"
- "${FLUXER_DEV_PROXY_PORT:-8088}:8088"
- "${FLUXER_DEV_APP_PROXY_PORT:-8080}:8080"
- "${FLUXER_DEV_API_PORT:-8771}:8771"
- "${FLUXER_DEV_GATEWAY_PORT:-8082}:8082"
- "${FLUXER_DEV_MARKETING_PORT:-3010}:3010"
- "${FLUXER_DEV_ADMIN_PORT:-3020}:3020"
- "${FLUXER_DEV_RUST_SERVICE_PORTS:-8100-8125}:8100-8125"
- "${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333"
- "${FLUXER_DEV_SEAWEEDFS_FILER_PORT:-8888}:8888"
- "${FLUXER_DEV_SEAWEEDFS_MASTER_PORT:-9333}:9333"
- "${FLUXER_DEV_SEAWEEDFS_VOLUME_PORT:-9340}:9340"
- "${FLUXER_DEV_SEAWEEDFS_ADMIN_PORT:-23646}:23646"
- "127.0.0.1:${FLUXER_DEV_RSPACK_PORT:-3000}:3000"
- "127.0.0.1:${FLUXER_DEV_PROXY_PORT:-8088}:8088"
- "127.0.0.1:${FLUXER_DEV_API_PORT:-8080}:8080"
- "127.0.0.1:${FLUXER_DEV_GATEWAY_PORT:-8771}:8771"
- "127.0.0.1:${FLUXER_DEV_MEDIA_PROXY_PORT:-8082}:8082"
- "127.0.0.1:${FLUXER_DEV_APP_PROXY_PORT:-8773}:8773"
- "127.0.0.1:${FLUXER_DEV_MARKETING_PORT:-3010}:3010"
- "127.0.0.1:${FLUXER_DEV_ADMIN_PORT:-3020}:3020"
- "127.0.0.1:${FLUXER_DEV_SEAWEEDFS_S3_PORT:-3900}:8333"
depends_on:
- postgres
- valkey
- nats
- livekit
- meilisearch
- mailpit
postgres:
condition: service_healthy
valkey:
condition: service_started
nats:
condition: service_started
livekit:
condition: service_started
meilisearch:
condition: service_healthy
mailpit:
condition: service_started
extra_hosts:
- "host.docker.internal:host-gateway"
cassandra:
image: cassandra:5.0.8
profiles:
- full
environment:
CASSANDRA_CLUSTER_NAME: fluxer-dev
CASSANDRA_DC: datacenter1
CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch
HEAP_NEWSIZE: 128M
MAX_HEAP_SIZE: 768M
volumes:
- cassandra-data:/var/lib/cassandra
ports:
- "${FLUXER_DEV_CASSANDRA_PORT:-9042}:9042"
postgres:
image: postgres:16-alpine
environment:
@@ -284,13 +283,19 @@ services:
volumes:
- postgres-data:/var/lib/postgresql/data
ports:
- "${FLUXER_DEV_POSTGRES_PORT:-5432}:5432"
- "127.0.0.1:${FLUXER_DEV_POSTGRES_PORT:-5432}:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U fluxer -d fluxer"]
interval: 2s
timeout: 5s
retries: 30
start_period: 5s
valkey:
image: valkey/valkey:8.1.7-alpine
command: ["valkey-server", "--save", "", "--appendonly", "no"]
ports:
- "${FLUXER_DEV_VALKEY_PORT:-6379}:6379"
- "127.0.0.1:${FLUXER_DEV_VALKEY_PORT:-6379}:6379"
nats:
image: nats:2.14.2-alpine
@@ -298,33 +303,22 @@ services:
volumes:
- nats-data:/data
ports:
- "${FLUXER_DEV_NATS_PORT:-4222}:4222"
- "${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222"
- "127.0.0.1:${FLUXER_DEV_NATS_PORT:-4222}:4222"
- "127.0.0.1:${FLUXER_DEV_NATS_MONITOR_PORT:-8222}:8222"
livekit:
image: livekit/livekit-server:v1.12.0
command: ["--config", "/etc/livekit.yaml", "--bind", "0.0.0.0"]
environment:
LIVEKIT_RTC_TCP_PORT: "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}"
LIVEKIT_RTC_UDP_PORT_START: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}"
LIVEKIT_RTC_UDP_PORT_END: "${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}"
volumes:
- ./livekit.yaml:/etc/livekit.yaml:ro
ports:
- "${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880"
- "${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:7881"
- "${FLUXER_DEV_LIVEKIT_UDP_PORTS:-7882-7892}:7882-7892/udp"
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:9.3.2
profiles:
- full
environment:
discovery.type: single-node
xpack.security.enabled: "true"
xpack.security.http.ssl.enabled: "false"
ELASTIC_PASSWORD: fluxer-dev-elasticsearch
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
volumes:
- elasticsearch-data:/usr/share/elasticsearch/data
ports:
- "${FLUXER_DEV_ELASTICSEARCH_PORT:-9200}:9200"
- "127.0.0.1:${FLUXER_DEV_LIVEKIT_PORT:-7880}:7880"
- "127.0.0.1:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}:${FLUXER_DEV_LIVEKIT_TCP_PORT:-7881}"
- "127.0.0.1:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}:${FLUXER_DEV_LIVEKIT_UDP_PORT:-7882}/udp"
meilisearch:
image: getmeili/meilisearch:v1.12
@@ -334,7 +328,13 @@ services:
volumes:
- meilisearch-data:/meili_data
ports:
- "${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700"
- "127.0.0.1:${FLUXER_DEV_MEILISEARCH_PORT:-7700}:7700"
healthcheck:
test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:7700/health"]
interval: 2s
timeout: 5s
retries: 30
start_period: 5s
mailpit:
image: axllent/mailpit:v1.30
@@ -349,6 +349,7 @@ services:
volumes:
pnpm-store:
docs-venv:
root-node-modules:
fluxer-api-node-modules:
fluxer-app-node-modules:
@@ -394,9 +395,7 @@ volumes:
cargo-registry:
cargo-git:
rust-target:
cassandra-data:
nats-data:
elasticsearch-data:
meilisearch-data:
mailpit-data:
postgres-data:
.devcontainer/fix-docker-socket.sh
+6 -26
View File
@@ -1,10 +1,10 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -uo pipefail
set -euo pipefail
SOCKET="${DOCKER_SOCKET:-/var/run/docker.sock}"
USER_NAME="${USER:-vscode}"
USER_NAME="$(id -un)"
if [ ! -S "$SOCKET" ]; then
echo "fix-docker-socket: no socket at $SOCKET; skipping (Docker-in-devcontainer will not work)"
@@ -16,31 +16,11 @@ if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then
exit 0
fi
socket_gid="$(stat -c '%g' "$SOCKET" 2>/dev/null || echo "")"
if [ -z "$socket_gid" ]; then
echo "fix-docker-socket: could not stat $SOCKET; skipping" >&2
exit 0
fi
sudo sh -c '
set -e
gid="$1"
user="$2"
socket="$3"
if ! getent group "$gid" >/dev/null 2>&1; then
groupadd --gid "$gid" docker-host
fi
group_name="$(getent group "$gid" | cut -d: -f1)"
usermod --append --groups "$group_name" "$user"
chgrp "$gid" "$socket"
chmod g+rw "$socket"
' sh "$socket_gid" "$USER_NAME" "$SOCKET" || {
echo "fix-docker-socket: could not adjust $SOCKET; run docker with sudo" >&2
exit 0
}
sudo setfacl --modify "user:${USER_NAME}:rw" "$SOCKET"
if docker version --format '{{.Server.Version}}' >/dev/null 2>&1; then
echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME (gid $socket_gid)"
echo "fix-docker-socket: $SOCKET is now usable as $USER_NAME"
else
echo "fix-docker-socket: $SOCKET still unreachable as $USER_NAME; run docker with sudo" >&2
echo "fix-docker-socket: $SOCKET is still unreachable as $USER_NAME" >&2
exit 1
fi
.devcontainer/fix-workspace-permissions.sh
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -euo pipefail
owner="$(id -u):$(id -g)"
repair_tree() {
local path="$1"
local unwritable
if [ ! -d "$path" ]; then
echo "fix-workspace-permissions: expected mount is missing: $path" >&2
exit 1
fi
unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)"
if [ ! -w "$path" ] || [ -n "$unwritable" ]; then
sudo find "$path" -xdev \( -type d -o -type f \) -exec chown "$owner" {} +
fi
unwritable="$(find "$path" -xdev \( -type d -o -type f \) ! -writable -print -quit 2>/dev/null || true)"
if [ ! -w "$path" ] || [ -n "$unwritable" ]; then
echo "fix-workspace-permissions: $path is not writable as $(id -un)" >&2
exit 1
fi
}
for path in \
/workspaces/fluxer/target \
/home/vscode/.cargo/registry \
/home/vscode/.cargo/git \
/home/vscode/.local \
/home/vscode/.local/share/pnpm/store \
/workspaces/fluxer/fluxer_docs/.venv; do
repair_tree "$path"
done
while IFS= read -r -d '' path; do
if mountpoint -q "$path"; then
repair_tree "$path"
fi
done < <(find /workspaces/fluxer -maxdepth 4 -type d -name node_modules -prune -print0)
.devcontainer/livekit.yaml
+1 -2
View File
@@ -1,11 +1,10 @@
port: 7880
keys:
devkey: secret
devkey: fluxer-livekit-development-secret
rtc:
tcp_port: 7881
udp_port: 7882-7892
node_ip: 127.0.0.1
use_mdns: true
stun_servers:
config/env/development.env
Vendored
+2 -7
View File
@@ -30,12 +30,6 @@ FLUXER_POSTGRES_PASSWORD=fluxer
FLUXER_POSTGRES_SSL=false
FLUXER_POSTGRES_MAX_CONNECTIONS=20
FLUXER_POSTGRES_KV_TABLE=fluxer_kv
FLUXER_CASSANDRA_HOSTS=cassandra
FLUXER_CASSANDRA_PORT=9042
FLUXER_CASSANDRA_KEYSPACE=fluxer
FLUXER_CASSANDRA_LOCAL_DC=datacenter1
FLUXER_CASSANDRA_USERNAME=fluxer
FLUXER_CASSANDRA_PASSWORD=fluxer
FLUXER_KV_URL=redis://valkey:6379/0
FLUXER_NATS_URL=nats://nats:4222
FLUXER_NATS_JETSTREAM_URL=nats://nats:4222
@@ -66,8 +60,9 @@ FLUXER_S3_BUCKET_STATIC=fluxer-static
FLUXER_LIVEKIT_ENABLED=true
FLUXER_LIVEKIT_URL=ws://localhost:8088/livekit
FLUXER_LIVEKIT_INTERNAL_URL=http://localhost:7880
FLUXER_LIVEKIT_API_KEY=devkey
FLUXER_LIVEKIT_API_SECRET=secret
FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secret
FLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekit
FLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686}
deploy/self-hosting/docker-compose.yml
+1
View File
@@ -47,6 +47,7 @@ x-fluxer-env: &fluxer-env
FLUXER_LIVEKIT_ENABLED: "true"
FLUXER_LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}
FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}
FLUXER_LIVEKIT_INTERNAL_URL: http://livekit:7880
FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit
FLUXER_LIVEKIT_DEFAULT_REGION: '{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}'
fluxer_api/src/api/App.ts
+1 -1
View File
@@ -39,7 +39,7 @@ function AbuseAwareAppErrorHandler(err: Error, ctx: Context<HonoEnv>): Response
export async function createAPIApp(options: CreateAPIAppOptions): Promise<APIAppResult> {
const {config, logger} = options;
const shutdownApiLifecycle = createShutdown(logger);
const shutdownApiLifecycle = createShutdown(config, logger);
setIsDevelopment(config.nodeEnv === 'development');
const routes = new Hono<HonoEnv>({strict: true});
configureMiddleware(routes, {
fluxer_api/src/api/Config.ts
+1
View File
@@ -306,6 +306,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
apiSecret: master.integrations.voice.api_secret,
webhookUrl: master.integrations.voice.webhook_url,
url: master.integrations.voice.url,
internalUrl: master.integrations.voice.internal_url,
defaultRegion: master.integrations.voice.default_region,
},
stripe: {
fluxer_api/src/api/app/APILifecycle.ts
+28 -15
View File
@@ -36,6 +36,11 @@ import {JetStreamWorkerQueue} from '../worker/JetStreamWorkerQueue';
import {WorkerService} from '../worker/WorkerService';
let jsConnectionManager: JetStreamConnectionManager | null = null;
function unsupportedDatabaseBackend(backend: never): never {
throw new Error(`Unsupported database backend during shutdown: ${String(backend)}`);
}
export function createInitializer(config: APIConfig, logger: ILogger): () => Promise<void> {
return async (): Promise<void> => {
try {
@@ -131,7 +136,7 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro
try {
const kvDeletionQueue = getKVAccountDeletionQueue();
if (await kvDeletionQueue.needsRebuild()) {
logger.warn('KV deletion queue needs rebuild, rebuilding...');
logger.info('KV deletion queue needs rebuild, rebuilding...');
await kvDeletionQueue.rebuildState();
} else {
logger.info('KV deletion queue state is healthy');
@@ -204,13 +209,13 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro
logger.info('API service initialization complete');
} catch (error) {
logger.error({error}, 'API service initialization failed');
await createShutdown(logger)();
await createShutdown(config, logger)();
throw error;
}
};
}
export function createShutdown(logger: ILogger): () => Promise<void> {
export function createShutdown(config: APIConfig, logger: ILogger): () => Promise<void> {
return async (): Promise<void> => {
logger.info('Shutting down API service...');
if (jsConnectionManager) {
@@ -258,18 +263,26 @@ export function createShutdown(logger: ILogger): () => Promise<void> {
} catch (error) {
logger.error({error}, 'Error shutting down report service');
}
try {
setDatabaseQueryExecutor(null);
await shutdownPostgres();
logger.info('Postgres client shut down');
} catch (error) {
logger.error({error}, 'Error shutting down Postgres client');
}
try {
await shutdownCassandra();
logger.info('Cassandra client shut down');
} catch (error) {
logger.error({error}, 'Error shutting down Cassandra client');
setDatabaseQueryExecutor(null);
switch (config.database.backend) {
case 'postgres':
try {
await shutdownPostgres();
logger.info('Postgres client shut down');
} catch (error) {
logger.error({error}, 'Error shutting down Postgres client');
}
break;
case 'cassandra':
try {
await shutdownCassandra();
logger.info('Cassandra client shut down');
} catch (error) {
logger.error({error}, 'Error shutting down Cassandra client');
}
break;
default:
unsupportedDatabaseBackend(config.database.backend);
}
logger.info('API service shutdown complete');
};
fluxer_api/src/api/app/ControllerRegistry.ts
+3 -1
View File
@@ -66,7 +66,9 @@ export function registerControllers(routes: HonoApp, config: APIConfig): void {
TestHarnessController(routes);
}
UserController(routes);
registerInboundSmsWebhook(routes);
if (config.sms.enabled) {
registerInboundSmsWebhook(routes);
}
WebhookController(routes);
OAuth2Controller(routes);
OAuth2ApplicationsController(routes);
fluxer_api/src/api/config/APIConfig.ts
+1
View File
@@ -204,6 +204,7 @@ export interface APIConfig {
apiSecret?: string;
webhookUrl?: string;
url?: string;
internalUrl?: string;
defaultRegion?: {
id: string;
name: string;
fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
+27 -19
View File
@@ -19,6 +19,8 @@ interface PageState {
const VALUE_SEPARATOR = '\u001f';
const ENCODED_TYPE_KEY = '__fluxer_type';
const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;
const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';
function normalizeCql(cql: string): string {
return cql.replace(/\s+/g, ' ').trim();
@@ -354,8 +356,13 @@ function parseEqWhere(whereSql: string, cql: string): ReadonlyArray<EqWhereExpr>
}
export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<void> {
const table = quoteIdentifier(client.kvTable());
await client.query(`
const kvTable = client.kvTable();
const table = quoteIdentifier(kvTable);
await client.transaction(async (db) => {
await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_LOCK_TIMEOUT]);
await db.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, kvTable]);
await db.query("SELECT set_config('statement_timeout', '0', true)");
await db.query(`
CREATE TABLE IF NOT EXISTS ${table} (
table_name text NOT NULL,
partition_key text NOT NULL,
@@ -365,28 +372,29 @@ CREATE TABLE IF NOT EXISTS ${table} (
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (table_name, row_key)
)`);
await client.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,
);
await client.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,
);
await client.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,
);
await client.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,
);
await client.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${client.kvTable()}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
);
await client.query(`
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
);
await db.query(`
UPDATE ${table}
SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)
WHERE table_name = 'messages'
AND partition_key = row_key
AND split_part(row_key, chr(31), 3) <> ''`);
await client.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${client.kvTable()}_partition_idx`)}`);
await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);
});
}
export async function pruneExpiredPostgresKvRows(client: IPostgresClient, batchSize = 5000): Promise<number> {
fluxer_api/src/api/infrastructure/KVAccountDeletionQueueService.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export class KVAccountDeletionQueueService {
}
async rebuildState(): Promise<void> {
Logger.info('Starting deletion queue rebuild from Cassandra');
Logger.info('Starting deletion queue rebuild from primary database');
try {
await this.kvClient.del(QUEUE_KEY);
await this.kvClient.del(STATE_VERSION_KEY);
fluxer_api/src/api/infrastructure/LiveKitService.ts
+14 -1
View File
@@ -109,6 +109,18 @@ function createRoomServiceClient(endpoint: string, apiKey: string, apiSecret: st
return client;
}
function resolveRoomServiceEndpoint(server: VoiceServerRecord): string {
const defaultRegion = Config.voice.defaultRegion;
const internalUrl = Config.voice.internalUrl;
if (!defaultRegion || !internalUrl) {
return server.endpoint;
}
if (server.regionId !== defaultRegion.id || server.serverId !== `${defaultRegion.id}-server-1`) {
return server.endpoint;
}
return internalUrl;
}
export class LiveKitService extends ILiveKitService {
private serverClients: Map<string, Map<string, ServerClientConfig>> = new Map();
private topology: VoiceTopology;
@@ -453,12 +465,13 @@ export class LiveKitService extends ILiveKitService {
const servers = this.topology.getServersForRegion(region.id);
const serverMap: Map<string, ServerClientConfig> = new Map();
for (const server of servers) {
const roomServiceEndpoint = resolveRoomServiceEndpoint(server);
serverMap.set(server.serverId, {
endpoint: server.endpoint,
apiKey: server.apiKey,
apiSecret: server.apiSecret,
isActive: server.isActive,
roomServiceClient: createRoomServiceClient(server.endpoint, server.apiKey, server.apiSecret),
roomServiceClient: createRoomServiceClient(roomServiceEndpoint, server.apiKey, server.apiSecret),
});
}
newMap.set(region.id, serverMap);
fluxer_api/src/api/middleware/ServiceMiddleware.ts
+1 -1
View File
@@ -284,7 +284,7 @@ export function setInjectedAccountPolicyEvaluator(evaluator: IAccountPolicyEvalu
function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
if (_registrationRiskEvaluator) return _registrationRiskEvaluator;
if (!Config.risk.enabled) {
Logger.warn(
Logger.info(
{},
'[ServiceMiddleware] integrations.risk_integration.enabled is false — account risk scoring is disabled',
);
fluxer_app_proxy/src/routes/assets_proxy.rs
+1 -1
View File
@@ -148,7 +148,7 @@ pub async fn proxy_assets(
response
}
async fn serve_local_asset(
pub(super) async fn serve_local_asset(
static_dir: &str,
relative_path: &str,
request_headers: &HeaderMap,
fluxer_app_proxy/src/routes/spa_index.rs
+25
View File
@@ -19,6 +19,7 @@ use axum::{
use std::path::Path;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use super::assets_proxy::serve_local_asset;
use super::spa_static::{CORS_ALLOW_ANY_VALUE, guess_mime, is_font_mime};
const ACCEPT_CH_VALUE: &str = "DPR, Sec-CH-DPR, Sec-CH-Width, Save-Data, ECT, Downlink";
@@ -35,6 +36,14 @@ pub async fn spa_catch_all(
if let Some(cache_control) = static_root_file_cache_control(request_path) {
return serve_static_file(&state.config.static_dir, request_path, cache_control).await;
}
if is_static_asset_path(request_path) {
return serve_local_asset(
&state.config.static_dir,
request_path.trim_start_matches('/'),
&headers,
)
.await;
}
serve_spa_index(&state, &headers, request_path).await
}
@@ -42,6 +51,16 @@ pub async fn spa_catch_all(
const CRAWL_CONTROL_CACHE_CONTROL: &str = "public, max-age=300, must-revalidate";
const STATIC_ROOT_FILES: &[(&str, &str)] = &[("/robots.txt", CRAWL_CONTROL_CACHE_CONTROL)];
const STATIC_ASSET_PREFIXES: &[&str] = &[
"/avatars/",
"/badges/",
"/desktop/",
"/embeds/",
"/emoji/",
"/libs/",
"/marketing/",
"/web/",
];
fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> {
STATIC_ROOT_FILES
@@ -50,6 +69,12 @@ fn static_root_file_cache_control(request_path: &str) -> Option<&'static str> {
.map(|(_, cache_control)| *cache_control)
}
fn is_static_asset_path(request_path: &str) -> bool {
STATIC_ASSET_PREFIXES
.iter()
.any(|prefix| request_path.starts_with(prefix))
}
async fn serve_static_file(
static_dir: &str,
request_path: &str,
fluxer_docs/serve.sh
+28 -22
View File
@@ -3,6 +3,7 @@ set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VENV="$HERE/.venv"
REQUIREMENTS_STAMP="$VENV/.requirements.sha256"
ADDR="${ZENSICAL_DEV_ADDR:-0.0.0.0:8000}"
LOG="${ZENSICAL_LOG:-/tmp/zensical-serve.log}"
PORT="${ADDR##*:}"
@@ -14,29 +15,34 @@ ensure_env() {
python3 -m venv "$VENV"
"$VENV/bin/python" -m pip install --quiet --upgrade pip
fi
"$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt"
python_fingerprint="$(python3 -c 'import platform, sys; print(f"{sys.implementation.cache_tag}:{platform.machine()}")')"
requirements_hash="$(printf '%s\0%s\n' "$python_fingerprint" "$(sha256sum "$HERE/requirements.txt" | cut -d' ' -f1)" | sha256sum | cut -d' ' -f1)"
if [ ! -f "$REQUIREMENTS_STAMP" ] || [ "$(cat "$REQUIREMENTS_STAMP")" != "$requirements_hash" ]; then
"$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt"
printf '%s\n' "$requirements_hash" >"$REQUIREMENTS_STAMP"
fi
}
case "${1:-serve}" in
--bootstrap)
ensure_env
;;
--daemon)
ensure_env
if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then
echo "zensical already serving on ${ADDR}"
exit 0
fi
setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 &
disown 2>/dev/null || true
echo "zensical serving on ${ADDR} (logs: ${LOG})"
;;
serve)
ensure_env
exec "$VENV/bin/zensical" serve -a "$ADDR"
;;
*)
ensure_env
exec "$VENV/bin/zensical" "$@"
;;
--bootstrap)
ensure_env
;;
--daemon)
ensure_env
if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then
echo "zensical already serving on ${ADDR}"
exit 0
fi
setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 &
disown 2>/dev/null || true
echo "zensical serving on ${ADDR} (logs: ${LOG})"
;;
serve)
ensure_env
exec "$VENV/bin/zensical" serve -a "$ADDR"
;;
*)
ensure_env
exec "$VENV/bin/zensical" "$@"
;;
esac
fluxer_gateway/Dockerfile
+2 -9
View File
@@ -12,22 +12,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
git \
curl \
make \
cmake \
gcc \
g++ \
libc6-dev \
libssl-dev \
libuv1-dev \
zlib1g-dev \
pkg-config \
gettext-base \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \
RUN curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://github.com/erlang/rebar3/releases/download/3.24.0/rebar3 -o /usr/local/bin/rebar3 && \
chmod +x /usr/local/bin/rebar3
RUN curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \
RUN curl --retry 5 --retry-delay 2 --retry-all-errors --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs | \
sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}"
COPY . .
@@ -47,8 +42,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
libstdc++6 \
libuv1t64 \
zlib1g \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /usr/src/app/fluxer_gateway/_build/prod/rel/fluxer_gateway .
fluxer_gateway/rebar.config
+1 -3
View File
@@ -6,7 +6,6 @@
{jose, "1.11.10"},
{ezstd, "1.1.0"},
{enats, "1.2.0"},
{erlcass, "4.1.3"},
{eqwalizer_support,
{git_subdir, "https://github.com/whatsapp/eqwalizer.git",
{ref, "c57aa9e3a05553d1ba66ea647a2177eaa469211b"}, "eqwalizer_support"}}
@@ -32,7 +31,6 @@
{relx, [
{release, {fluxer_gateway, "0.0.0"}, [
fluxer_gateway,
{erlcass, load},
enats,
sasl
]},
@@ -90,7 +88,7 @@
]},
{plt_apps, all_deps},
{plt_extra_apps, [
crypto, enats, erlcass, inets, jose, public_key, ranch, ssl
crypto, enats, inets, jose, public_key, ranch, ssl
]},
{warnings_file, "dialyzer.ignore-warnings"}
]}.
fluxer_gateway/rebar.lock
-3
View File
@@ -9,7 +9,6 @@
{ref,"c57aa9e3a05553d1ba66ea647a2177eaa469211b"},
"eqwalizer_support"},
0},
{<<"erlcass">>,{pkg,<<"erlcass">>,<<"4.1.3">>},0},
{<<"ezstd">>,{pkg,<<"ezstd">>,<<"1.1.0">>},0},
{<<"jose">>,{pkg,<<"jose">>,<<"1.11.10">>},0},
{<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.4.0">>},1},
@@ -21,7 +20,6 @@
{<<"cowlib">>, <<"54592074EBBBB92EE4746C8A8846E5605052F29309D3A873468D76CDF932076F">>},
{<<"enats">>, <<"D7459C804013CAFA4AF880B18D446C48890D28D372D62AD66C76187E5779248D">>},
{<<"enats_msg">>, <<"50631124F37D88BE76A91A5B96A6565C5981EBF917CD819F0175CA658A966F43">>},
{<<"erlcass">>, <<"17AC8F39B0A4703B12A8994663920993E191757831E125CF2C7EFB267DB66E7D">>},
{<<"ezstd">>, <<"D3B483D6ACFADFB65DBA4015371E6D54526DBF3D9EF0941B5ADD8BF5890731F4">>},
{<<"jose">>, <<"A903F5227417BD2A08C8A00A0CBCC458118BE84480955E8D251297A425723F83">>},
{<<"opentelemetry_api">>, <<"63CA1742F92F00059298F478048DFB826F4B20D49534493D6919A0DB39B6DB04">>},
@@ -32,7 +30,6 @@
{<<"cowlib">>, <<"7F478D80D66B747344F0EA7708C187645CFCC08B11AA424632F78E25BF05DB51">>},
{<<"enats">>, <<"20DEB3CB1D3E960194DF8B136C40D2DB085B485BBA5E493B340AB5F9FD2BED22">>},
{<<"enats_msg">>, <<"C4F2139E5144FABC99AFE01B8B016AD9DA278CDDC60857AC5D5AFB0AD1283534">>},
{<<"erlcass">>, <<"37237C12ABE5745E0F900180D0ADB129634619BD6B2F57995BA020E63DDC6BEC">>},
{<<"ezstd">>, <<"28CFA0ED6CC3922095AD5BA0F23392A1664273358B17184BAA909868361184E7">>},
{<<"jose">>, <<"0D6CD36FF8BA174DB29148FC112B5842186B68A90CE9FC2B3EC3AFE76593E614">>},
{<<"opentelemetry_api">>, <<"3DFBBFAA2C2ED3121C5C483162836C4F9027DEF469C41578AF5EF32589FCFC58">>},
fluxer_gateway/src/gateway/fluxer_gateway_config.erl
+1 -45
View File
@@ -59,7 +59,7 @@ env_services_config() ->
-spec env_gateway_config() -> map().
env_gateway_config() ->
maps:merge(env_gateway_base_config(), env_gateway_hotpatch_config()).
env_gateway_base_config().
-spec env_gateway_base_config() -> map().
env_gateway_base_config() ->
@@ -107,36 +107,6 @@ env_gateway_base_config() ->
)
}.
-spec env_gateway_hotpatch_config() -> map().
env_gateway_hotpatch_config() ->
#{
<<"hotpatch_enabled">> => env_bool("FLUXER_GATEWAY_HOTPATCH_ENABLED", false),
<<"hotpatch_cassandra_hosts">> => env_optional_binary(
"FLUXER_GATEWAY_HOTPATCH_CASSANDRA_HOSTS"
),
<<"hotpatch_cassandra_port">> => env_int(
"FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PORT", 9042
),
<<"hotpatch_cassandra_keyspace">> => env_binary(
"FLUXER_GATEWAY_HOTPATCH_CASSANDRA_KEYSPACE", <<"fluxer">>
),
<<"hotpatch_cassandra_username">> => env_optional_binary(
"FLUXER_GATEWAY_HOTPATCH_CASSANDRA_USERNAME"
),
<<"hotpatch_cassandra_password">> => env_optional_binary(
"FLUXER_GATEWAY_HOTPATCH_CASSANDRA_PASSWORD"
),
<<"hotpatch_public_keys">> => env_optional_binary(
"FLUXER_GATEWAY_HOTPATCH_PUBLIC_KEYS"
),
<<"hotpatch_poll_interval_ms">> => env_int(
"FLUXER_GATEWAY_HOTPATCH_POLL_INTERVAL_MS", 5000
),
<<"hotpatch_startup_sync_timeout_ms">> => env_int(
"FLUXER_GATEWAY_HOTPATCH_STARTUP_SYNC_TIMEOUT_MS", 30000
)
}.
-spec env_nats_config() -> map().
env_nats_config() ->
#{
@@ -295,20 +265,6 @@ build_http_config(Service) ->
),
gateway_http_recovery_timeout_ms =>
get_int(Service, <<"gateway_http_recovery_timeout_ms">>, 15000),
hotpatch_enabled => get_bool(Service, <<"hotpatch_enabled">>, false),
hotpatch_cassandra_hosts =>
optional_string(get_optional_binary(Service, <<"hotpatch_cassandra_hosts">>)),
hotpatch_cassandra_port => get_int(Service, <<"hotpatch_cassandra_port">>, 9042),
hotpatch_cassandra_keyspace =>
get_binary(Service, <<"hotpatch_cassandra_keyspace">>, <<"fluxer">>),
hotpatch_cassandra_username =>
get_optional_binary(Service, <<"hotpatch_cassandra_username">>),
hotpatch_cassandra_password =>
get_optional_binary(Service, <<"hotpatch_cassandra_password">>),
hotpatch_public_keys => get_optional_binary(Service, <<"hotpatch_public_keys">>),
hotpatch_poll_interval_ms => get_int(Service, <<"hotpatch_poll_interval_ms">>, 5000),
hotpatch_startup_sync_timeout_ms =>
get_int(Service, <<"hotpatch_startup_sync_timeout_ms">>, 30000),
gateway_http_cleanup_interval_ms =>
get_int(Service, <<"gateway_http_cleanup_interval_ms">>, 30000),
gateway_http_cleanup_max_age_ms =>
fluxer_gateway/src/gateway/fluxer_gateway_sup.erl
+1 -2
View File
@@ -38,8 +38,7 @@ common_children() ->
child_spec(gateway_nats_pool, gateway_nats_pool),
child_spec(gateway_event_pause, gateway_event_pause),
child_spec(gateway_concurrency, gateway_concurrency),
child_spec(gateway_rollout_config, gateway_rollout_config),
child_spec(gateway_hotpatch_reconciler, gateway_hotpatch_reconciler)
child_spec(gateway_rollout_config, gateway_rollout_config)
] ++ cluster_children() ++
[
child_spec(gateway_dispatch_relay, gateway_dispatch_relay),
fluxer_gateway/src/gateway/gateway_hotpatch_bundle.erl
-292
View File
@@ -1,292 +0,0 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_hotpatch_bundle).
-typing([eqwalizer]).
-export([
compress_term/1,
decompress_term/1,
bundle_hash/1,
signing_payload/1,
sign/2,
verify_signature/4,
parse_public_keys/1,
decode_signed_event/2
]).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.
-define(DOMAIN, <<"fluxer-gateway-hotpatch-v1">>).
-define(ZSTD_LEVEL, 3).
-spec compress_term(term()) -> {ok, binary()} | {error, term()}.
compress_term(Term) ->
compress_binary(term_to_binary(Term, [deterministic])).
-spec decompress_term(binary()) -> {ok, term()} | {error, term()}.
decompress_term(Compressed) when is_binary(Compressed) ->
case decompress_binary(Compressed) of
{ok, Binary} -> decode_term(Binary);
{error, Reason} -> {error, Reason}
end.
-spec decode_term(binary()) -> {ok, term()} | {error, term()}.
decode_term(Binary) ->
try
{ok, binary_to_term(Binary, [safe])}
catch
Class:Reason -> {error, {invalid_term, Class, Reason}}
end.
-spec bundle_hash(binary()) -> binary().
bundle_hash(CompressedBundle) when is_binary(CompressedBundle) ->
crypto:hash(sha256, CompressedBundle).
-spec signing_payload(binary()) -> binary().
signing_payload(CompressedBundle) when is_binary(CompressedBundle) ->
<<?DOMAIN/binary, 0, CompressedBundle/binary>>.
-spec sign(binary(), binary()) -> {ok, binary()} | {error, term()}.
sign(CompressedBundle, PrivateKey) when is_binary(CompressedBundle), is_binary(PrivateKey) ->
try
{ok, crypto:sign(eddsa, none, signing_payload(CompressedBundle), [PrivateKey, ed25519])}
catch
Class:Reason -> {error, {sign_failed, Class, Reason}}
end.
-spec verify_signature(binary(), binary(), binary(), #{binary() => binary()}) ->
ok | {error, term()}.
verify_signature(CompressedBundle, SignerKeyId, Signature, PublicKeys) when
is_binary(CompressedBundle),
is_binary(SignerKeyId),
is_binary(Signature),
is_map(PublicKeys)
->
case maps:get(SignerKeyId, PublicKeys, undefined) of
undefined ->
{error, {unknown_signer, SignerKeyId}};
PublicKey ->
verify_with_key(CompressedBundle, Signature, PublicKey)
end.
-spec parse_public_keys(term()) ->
{ok, #{binary() => binary()}} | {error, term()}.
parse_public_keys(undefined) ->
{ok, #{}};
parse_public_keys(Value) when is_list(Value) ->
parse_public_keys(type_conv:ensure_binary(Value));
parse_public_keys(Value) when is_binary(Value) ->
Tokens = [
string:trim(Token)
|| Token <- binary:split(Value, [<<",">>, <<"\n">>, <<";">>], [global]),
string:trim(Token) =/= <<>>
],
parse_public_key_tokens(Tokens, #{});
parse_public_keys(Value) ->
{error, {invalid_public_keys_config, Value}}.
-spec decode_signed_event(map(), #{binary() => binary()}) -> {ok, map()} | {error, term()}.
decode_signed_event(Event, PublicKeys) when is_map(Event), is_map(PublicKeys) ->
Bundle = event_value(bundle, Event),
SignerKeyId = event_value(signer_key_id, Event),
Signature = event_value(signature, Event),
BundleHash = event_value(bundle_sha256, Event),
decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys).
-spec decode_event_payload(term(), term(), term(), term(), #{binary() => binary()}) ->
{ok, map()} | {error, term()}.
decode_event_payload(Bundle, SignerKeyId, Signature, BundleHash, PublicKeys) when
is_binary(Bundle), is_binary(SignerKeyId), is_binary(Signature), is_binary(BundleHash)
->
case validate_event_hash(Bundle, BundleHash) of
ok -> decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys);
{error, Reason} -> {error, Reason}
end;
decode_event_payload(_Bundle, _SignerKeyId, _Signature, _BundleHash, _PublicKeys) ->
{error, invalid_event_payload}.
-spec decode_verified_event(binary(), binary(), binary(), #{binary() => binary()}) ->
{ok, map()} | {error, term()}.
decode_verified_event(Bundle, SignerKeyId, Signature, PublicKeys) ->
case verify_signature(Bundle, SignerKeyId, Signature, PublicKeys) of
ok -> decompress_bundle_map(Bundle);
{error, Reason} -> {error, Reason}
end.
-spec decompress_bundle_map(binary()) -> {ok, map()} | {error, term()}.
decompress_bundle_map(Bundle) ->
case decompress_term(Bundle) of
{ok, Map} when is_map(Map) -> {ok, Map};
{ok, Other} -> {error, {invalid_bundle_term, Other}};
{error, Reason} -> {error, Reason}
end.
-spec compress_binary(binary()) -> {ok, binary()} | {error, term()}.
compress_binary(Binary) ->
try erlang:apply(ezstd, compress, [Binary, ?ZSTD_LEVEL]) of
Compressed when is_binary(Compressed) -> {ok, Compressed};
{error, Reason} -> {error, {compress_failed, Reason}};
Other -> {error, {compress_failed, Other}}
catch
Class:Reason -> {error, {compress_failed, Class, Reason}}
end.
-spec decompress_binary(binary()) -> {ok, binary()} | {error, term()}.
decompress_binary(Binary) ->
try erlang:apply(ezstd, decompress, [Binary]) of
Decompressed when is_binary(Decompressed) -> {ok, Decompressed};
Decompressed when is_list(Decompressed) -> {ok, iolist_to_binary(Decompressed)};
{error, Reason} -> {error, {decompress_failed, Reason}};
Other -> {error, {decompress_failed, Other}}
catch
Class:Reason -> {error, {decompress_failed, Class, Reason}}
end.
-spec verify_with_key(binary(), binary(), binary()) -> ok | {error, term()}.
verify_with_key(CompressedBundle, Signature, PublicKey) when byte_size(PublicKey) =:= 32 ->
verify_with_valid_key(CompressedBundle, Signature, PublicKey);
verify_with_key(_CompressedBundle, _Signature, PublicKey) ->
{error, {invalid_public_key_size, byte_size(PublicKey)}}.
-spec verify_with_valid_key(binary(), binary(), binary()) -> ok | {error, term()}.
verify_with_valid_key(CompressedBundle, Signature, PublicKey) ->
try
crypto:verify(eddsa, none, signing_payload(CompressedBundle), Signature, [
PublicKey, ed25519
])
of
true -> ok;
false -> {error, invalid_signature}
catch
Class:Reason -> {error, {verify_failed, Class, Reason}}
end.
-spec parse_public_key_tokens([binary()], #{binary() => binary()}) ->
{ok, #{binary() => binary()}} | {error, term()}.
parse_public_key_tokens([], Acc) ->
{ok, Acc};
parse_public_key_tokens([Token | Rest], Acc) ->
case parse_public_key_token(Token) of
{ok, KeyId, PublicKey} -> parse_public_key_tokens(Rest, Acc#{KeyId => PublicKey});
{error, Reason} -> {error, Reason}
end.
-spec parse_public_key_token(binary()) -> {ok, binary(), binary()} | {error, term()}.
parse_public_key_token(Token) ->
case split_key_token(Token) of
{ok, KeyId, Encoded} -> parse_public_key_material(KeyId, Encoded);
error -> {error, {invalid_public_key_token, Token}}
end.
-spec parse_public_key_material(binary(), binary()) ->
{ok, binary(), binary()} | {error, term()}.
parse_public_key_material(KeyId, Encoded) ->
case decode_key_material(Encoded) of
{ok, PublicKey} when byte_size(PublicKey) =:= 32 -> {ok, KeyId, PublicKey};
{ok, PublicKey} -> {error, {invalid_public_key_size, KeyId, byte_size(PublicKey)}};
{error, Reason} -> {error, {invalid_public_key, KeyId, Reason}}
end.
-spec split_key_token(binary()) -> {ok, binary(), binary()} | error.
split_key_token(Token) ->
case binary:split(Token, <<":">>) of
[KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)};
_ -> split_key_token_equals(Token)
end.
-spec split_key_token_equals(binary()) -> {ok, binary(), binary()} | error.
split_key_token_equals(Token) ->
case binary:split(Token, <<"=">>) of
[KeyId, Encoded] -> {ok, string:trim(KeyId), string:trim(Encoded)};
_ -> error
end.
-spec decode_key_material(binary()) -> {ok, binary()} | {error, term()}.
decode_key_material(Encoded) ->
case try_base64(Encoded) of
{ok, Decoded} -> {ok, Decoded};
{error, _} -> try_base64url(Encoded)
end.
-spec try_base64(binary()) -> {ok, binary()} | {error, term()}.
try_base64(Encoded) ->
try
{ok, base64:decode(Encoded)}
catch
Class:Reason -> {error, {Class, Reason}}
end.
-spec try_base64url(binary()) -> {ok, binary()} | {error, term()}.
try_base64url(Encoded) ->
try
{ok, base64url:decode(Encoded)}
catch
Class:Reason -> {error, {Class, Reason}}
end.
-spec validate_event_hash(binary(), binary()) -> ok | {error, term()}.
validate_event_hash(Bundle, BundleHash) ->
case bundle_hash(Bundle) of
BundleHash -> ok;
Other -> {error, {bundle_hash_mismatch, Other, BundleHash}}
end.
-spec event_value(atom(), map()) -> term().
event_value(Key, Event) ->
maps:get(Key, Event, maps:get(atom_to_binary(Key, utf8), Event, undefined)).
-ifdef(TEST).
compress_decompress_roundtrip_test() ->
Term = #{
version => 1,
build_sha => <<"abc123">>,
modules => [#{module => <<"session_lifecycle">>, expected_current_md5 => <<0:128>>}]
},
{ok, Compressed} = compress_term(Term),
?assert(is_binary(Compressed)),
?assertEqual({ok, Term}, decompress_term(Compressed)).
sign_and_verify_roundtrip_test() ->
{PublicKey, PrivateKey} = ed25519_keypair(),
{ok, Compressed} = compress_term(#{version => 1}),
{ok, Signature} = sign(Compressed, PrivateKey),
Keys = #{<<"ops">> => PublicKey},
?assertEqual(ok, verify_signature(Compressed, <<"ops">>, Signature, Keys)),
?assertEqual(
{error, invalid_signature},
verify_signature(<<Compressed/binary, 0>>, <<"ops">>, Signature, Keys)
).
decode_signed_event_rejects_hash_mismatch_test() ->
{PublicKey, PrivateKey} = ed25519_keypair(),
{ok, Compressed} = compress_term(#{version => 1}),
{ok, Signature} = sign(Compressed, PrivateKey),
Event = #{
signer_key_id => <<"ops">>,
signature => Signature,
bundle_sha256 => <<0:256>>,
bundle => Compressed
},
?assertMatch(
{error, {bundle_hash_mismatch, _, _}},
decode_signed_event(Event, #{<<"ops">> => PublicKey})
).
parse_public_keys_test() ->
PublicKey = <<1:256>>,
Encoded = base64:encode(PublicKey),
?assertEqual(
{ok, #{<<"ops">> => PublicKey}}, parse_public_keys(<<"ops:", Encoded/binary>>)
).
ed25519_keypair() ->
{PublicKey, PrivateKey} = crypto:generate_key(eddsa, ed25519),
{require_binary(PublicKey), require_binary(PrivateKey)}.
require_binary(Value) when is_binary(Value) ->
Value.
-endif.
fluxer_gateway/src/gateway/gateway_hotpatch_cli.erl
-459
View File
@@ -1,459 +0,0 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_hotpatch_cli).
-typing([eqwalizer]).
-export([main/1, build_bundle/2, sign_bundle/3]).
-spec main([string()]) -> no_return().
main(Args) ->
halt(run_main(Args)).
-spec run_main([string()]) -> non_neg_integer().
run_main(["bundle", BuildSha | ModuleOrder]) when ModuleOrder =/= [] ->
write_json_result(build_bundle(type_conv:ensure_binary(BuildSha), ModuleOrder));
run_main(["sign", SignerKeyId, PrivateKeyPath, BundlePath]) ->
write_json_result(sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath));
run_main(["append", BuildSha, CreatedBy, EventPath]) ->
Result = append_event(
type_conv:ensure_binary(BuildSha), type_conv:ensure_binary(CreatedBy), EventPath
),
write_append_result(Result);
run_main(_Args) ->
write_usage(),
64.
-spec build_bundle(binary(), [string()]) -> {ok, map()} | {error, term()}.
build_bundle(BuildSha, ModuleSpecs) when is_binary(BuildSha), is_list(ModuleSpecs) ->
build_bundle_modules(ModuleSpecs, fun(Modules) ->
{ok, #{<<"version">> => 1, <<"build_sha">> => BuildSha, <<"modules">> => Modules}}
end).
-spec sign_bundle(binary(), file:filename(), binary()) -> {ok, map()} | {error, term()}.
sign_bundle(SignerKeyId, PrivateKeyPath, BundleJson) when
is_binary(SignerKeyId), is_binary(BundleJson)
->
case decode_json(BundleJson) of
{ok, Bundle} -> sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle);
{error, Reason} -> {error, Reason}
end.
-spec sign_decoded_bundle_with_key(binary(), file:filename(), term()) ->
{ok, map()} | {error, term()}.
sign_decoded_bundle_with_key(SignerKeyId, PrivateKeyPath, Bundle) ->
case read_private_key(PrivateKeyPath) of
{ok, PrivateKey} -> sign_decoded_bundle(SignerKeyId, Bundle, PrivateKey);
{error, Reason} -> {error, Reason}
end.
-spec sign_bundle_file(string(), file:filename(), file:filename()) ->
{ok, map()} | {error, term()}.
sign_bundle_file(SignerKeyId, PrivateKeyPath, BundlePath) ->
case file:read_file(BundlePath) of
{ok, BundleJson} ->
sign_bundle(type_conv:ensure_binary(SignerKeyId), PrivateKeyPath, BundleJson);
{error, Reason} ->
{error, {read_bundle_failed, Reason}}
end.
-spec sign_decoded_bundle(binary(), term(), binary()) -> {ok, map()} | {error, term()}.
sign_decoded_bundle(SignerKeyId, BundleJson, PrivateKey) ->
case json_to_bundle(BundleJson) of
{ok, Bundle} -> sign_bundle_term(SignerKeyId, Bundle, PrivateKey);
{error, Reason} -> {error, Reason}
end.
-spec sign_bundle_term(binary(), map(), binary()) -> {ok, map()} | {error, term()}.
sign_bundle_term(SignerKeyId, Bundle, PrivateKey) ->
case gateway_hotpatch_bundle:compress_term(Bundle) of
{ok, Compressed} -> sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey);
{error, Reason} -> {error, Reason}
end.
-spec sign_compressed_bundle(binary(), binary(), binary()) -> {ok, map()} | {error, term()}.
sign_compressed_bundle(SignerKeyId, Compressed, PrivateKey) ->
case gateway_hotpatch_bundle:sign(Compressed, PrivateKey) of
{ok, Signature} -> {ok, signed_event(SignerKeyId, Compressed, Signature)};
{error, Reason} -> {error, Reason}
end.
-spec signed_event(binary(), binary(), binary()) -> map().
signed_event(SignerKeyId, Compressed, Signature) ->
#{
<<"schema_version">> => 1,
<<"kind">> => <<"beam_bundle">>,
<<"created_by">> => gateway_hotpatch_runtime:node_name(),
<<"signer_key_id">> => SignerKeyId,
<<"bundle_sha256">> => encode_bytes(gateway_hotpatch_bundle:bundle_hash(Compressed)),
<<"signature">> => encode_bytes(Signature),
<<"bundle">> => encode_bytes(Compressed)
}.
-spec append_event(binary(), binary(), file:filename()) -> {ok, binary()} | {error, term()}.
append_event(BuildSha, CreatedBy, EventPath) ->
case file:read_file(EventPath) of
{ok, EventJson} -> append_event_json(BuildSha, CreatedBy, EventJson);
{error, Reason} -> {error, {read_event_failed, Reason}}
end.
-spec append_event_json(binary(), binary(), binary()) -> {ok, binary()} | {error, term()}.
append_event_json(BuildSha, CreatedBy, EventJson) ->
case decode_json(EventJson) of
{ok, EventJsonTerm} -> append_event_term(BuildSha, CreatedBy, EventJsonTerm);
{error, Reason} -> {error, Reason}
end.
-spec append_event_term(binary(), binary(), term()) -> {ok, binary()} | {error, term()}.
append_event_term(BuildSha, CreatedBy, EventJsonTerm) ->
case json_to_event(EventJsonTerm) of
{ok, Event} -> append_event_row(BuildSha, CreatedBy, Event);
{error, Reason} -> {error, Reason}
end.
-spec build_bundle_modules([string()], fun(([map()]) -> {ok, map()})) ->
{ok, map()} | {error, term()}.
build_bundle_modules(ModuleSpecs, Fun) ->
case build_bundle_module_list(ModuleSpecs, []) of
{ok, Modules} -> Fun(Modules);
{error, Reason} -> {error, Reason}
end.
-spec build_bundle_module_list([string()], [map()]) -> {ok, [map()]} | {error, term()}.
build_bundle_module_list([], Acc) ->
{ok, lists:reverse(Acc)};
build_bundle_module_list([ModuleSpec | Rest], Acc) ->
case bundle_module(ModuleSpec) of
{ok, Module} -> build_bundle_module_list(Rest, [Module | Acc]);
{error, Reason} -> {error, Reason}
end.
-spec bundle_module(string()) -> {ok, map()} | {error, term()}.
bundle_module(ModuleSpec) ->
case split_module_spec(ModuleSpec) of
{ok, ModuleName, BeamPath0} -> bundle_named_module(ModuleName, BeamPath0);
{error, Reason} -> {error, Reason}
end.
-spec bundle_named_module(string(), file:filename() | undefined) ->
{ok, map()} | {error, term()}.
bundle_named_module(ModuleName, BeamPath0) ->
case existing_module(ModuleName) of
{ok, Module} -> bundle_existing_module(Module, BeamPath0);
{error, Reason} -> {error, Reason}
end.
-spec bundle_existing_module(atom(), file:filename() | undefined) ->
{ok, map()} | {error, term()}.
bundle_existing_module(Module, BeamPath0) ->
case collect_bundle_parts(Module, BeamPath0) of
{ok, CurrentMd5, TargetMd5, CompressedBeam} ->
{ok, build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam)};
{error, Reason} ->
{error, Reason}
end.
-spec collect_bundle_parts(atom(), file:filename() | undefined) ->
{ok, binary(), binary(), binary()} | {error, term()}.
collect_bundle_parts(Module, BeamPath0) ->
case gateway_hotpatch_loader:current_md5(Module) of
{ok, CurrentMd5} -> collect_bundle_beam(Module, BeamPath0, CurrentMd5);
{error, Reason} -> {error, {current_md5_failed, Module, Reason}}
end.
-spec collect_bundle_beam(atom(), file:filename() | undefined, binary()) ->
{ok, binary(), binary(), binary()} | {error, term()}.
collect_bundle_beam(Module, BeamPath0, CurrentMd5) ->
case resolve_beam_path(Module, BeamPath0) of
{ok, BeamPath} -> read_bundle_beam(Module, BeamPath, CurrentMd5);
{error, Reason} -> {error, Reason}
end.
-spec read_bundle_beam(atom(), file:filename(), binary()) ->
{ok, binary(), binary(), binary()} | {error, term()}.
read_bundle_beam(Module, BeamPath, CurrentMd5) ->
case file:read_file(BeamPath) of
{ok, Beam} -> hash_bundle_beam(Module, CurrentMd5, Beam);
{error, Reason} -> {error, {read_beam_failed, Module, BeamPath, Reason}}
end.
-spec hash_bundle_beam(atom(), binary(), binary()) ->
{ok, binary(), binary(), binary()} | {error, term()}.
hash_bundle_beam(Module, CurrentMd5, Beam) ->
case gateway_hotpatch_loader:beam_md5(Beam) of
{ok, TargetMd5} -> compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam);
{error, Reason} -> {error, {target_md5_failed, Module, Reason}}
end.
-spec compress_bundle_beam(atom(), binary(), binary(), binary()) ->
{ok, binary(), binary(), binary()} | {error, term()}.
compress_bundle_beam(Module, CurrentMd5, TargetMd5, Beam) ->
case compress_beam(Beam) of
{ok, CompressedBeam} -> {ok, CurrentMd5, TargetMd5, CompressedBeam};
{error, Reason} -> {error, {beam_compress_failed, Module, Reason}}
end.
-spec build_module_entry(atom(), binary(), binary(), binary()) -> map().
build_module_entry(Module, CurrentMd5, TargetMd5, CompressedBeam) ->
#{
<<"module">> => atom_to_binary(Module, utf8),
<<"expected_current_md5">> => encode_bytes(CurrentMd5),
<<"target_md5">> => encode_bytes(TargetMd5),
<<"beam_zstd">> => encode_bytes(CompressedBeam)
}.
-spec append_event_row(binary(), binary(), map()) -> {ok, binary()} | {error, term()}.
append_event_row(BuildSha, CreatedBy, Event) ->
_ = fluxer_gateway_env:load(),
case gateway_hotpatch_store:connect() of
ok ->
gateway_hotpatch_store:append_event(
BuildSha,
CreatedBy,
maps:get(signer_key_id, Event),
maps:get(signature, Event),
maps:get(bundle_sha256, Event),
maps:get(bundle, Event)
);
{error, Reason} ->
{error, Reason}
end.
-spec split_module_spec(string()) ->
{ok, string(), file:filename() | undefined} | {error, term()}.
split_module_spec(ModuleSpec) ->
case string:split(ModuleSpec, "=", leading) of
[ModuleName, BeamPath] when ModuleName =/= "", BeamPath =/= "" ->
{ok, ModuleName, BeamPath};
[ModuleName] when ModuleName =/= "" ->
{ok, ModuleName, undefined};
_ ->
{error, {invalid_module_spec, ModuleSpec}}
end.
-spec existing_module(string()) -> {ok, atom()} | {error, term()}.
existing_module(ModuleName) ->
try
{ok, list_to_existing_atom(ModuleName)}
catch
error:badarg -> {error, {unknown_module, ModuleName}}
end.
-spec resolve_beam_path(atom(), file:filename() | undefined) ->
{ok, file:filename()} | {error, term()}.
resolve_beam_path(Module, undefined) ->
case code:which(Module) of
File when is_list(File) -> {ok, File};
Other -> {error, {module_beam_not_found, Module, Other}}
end;
resolve_beam_path(_Module, BeamPath) ->
{ok, BeamPath}.
-spec compress_beam(binary()) -> {ok, binary()} | {error, term()}.
compress_beam(Beam) ->
try erlang:apply(ezstd, compress, [Beam, 3]) of
Compressed when is_binary(Compressed) -> {ok, Compressed};
{error, Reason} -> {error, Reason};
Other -> {error, Other}
catch
Class:Reason -> {error, {Class, Reason}}
end.
-spec decode_json(binary()) -> {ok, term()} | {error, term()}.
decode_json(Json) ->
try
{ok, json:decode(Json)}
catch
Class:Reason -> {error, {decode_json_failed, Class, Reason}}
end.
-spec read_private_key(file:filename()) -> {ok, binary()} | {error, term()}.
read_private_key(Path) ->
case file:read_file(Path) of
{ok, PrivateKey} when byte_size(PrivateKey) =:= 32 ->
{ok, PrivateKey};
{ok, PrivateKey} ->
{error, {invalid_private_key_size, byte_size(PrivateKey)}};
{error, Reason} ->
{error, {read_private_key_failed, Reason}}
end.
-spec json_to_bundle(term()) -> {ok, map()} | {error, term()}.
json_to_bundle(#{
<<"version">> := Version, <<"build_sha">> := BuildSha, <<"modules">> := Modules
}) when is_list(Modules) ->
case json_to_module_entries(Modules, []) of
{ok, Entries} ->
{ok, #{
<<"version">> => Version,
<<"build_sha">> => BuildSha,
<<"modules">> => Entries
}};
{error, Reason} ->
{error, Reason}
end;
json_to_bundle(Other) ->
{error, {invalid_bundle_json, Other}}.
-spec json_to_module_entries([term()], [map()]) -> {ok, [map()]} | {error, term()}.
json_to_module_entries([], Acc) ->
{ok, lists:reverse(Acc)};
json_to_module_entries([Module | Rest], Acc) ->
case json_to_module_entry(Module) of
{ok, Entry} -> json_to_module_entries(Rest, [Entry | Acc]);
{error, Reason} -> {error, Reason}
end.
-spec json_to_module_entry(term()) -> {ok, map()} | {error, term()}.
json_to_module_entry(#{
<<"module">> := Module,
<<"expected_current_md5">> := ExpectedMd5,
<<"target_md5">> := TargetMd5,
<<"beam_zstd">> := BeamZstd
}) ->
decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd);
json_to_module_entry(Other) ->
{error, {invalid_module_entry_json, Other}}.
-spec decode_module_entry(term(), term(), term(), term()) ->
{ok, map()} | {error, term()}.
decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) when
is_binary(Module), is_binary(ExpectedMd5), is_binary(TargetMd5), is_binary(BeamZstd)
->
case decode_bytes(ExpectedMd5) of
{ok, ExpectedMd5Bytes} ->
decode_module_entry_target(Module, ExpectedMd5Bytes, TargetMd5, BeamZstd);
{error, Reason} ->
{error, Reason}
end;
decode_module_entry(Module, ExpectedMd5, TargetMd5, BeamZstd) ->
{error, {invalid_module_entry_json, {Module, ExpectedMd5, TargetMd5, BeamZstd}}}.
-spec decode_module_entry_target(binary(), binary(), binary(), binary()) ->
{ok, map()} | {error, term()}.
decode_module_entry_target(Module, ExpectedMd5, TargetMd5, BeamZstd) ->
case decode_bytes(TargetMd5) of
{ok, TargetMd5Bytes} ->
decode_module_entry_beam(Module, ExpectedMd5, TargetMd5Bytes, BeamZstd);
{error, Reason} ->
{error, Reason}
end.
-spec decode_module_entry_beam(binary(), binary(), binary(), binary()) ->
{ok, map()} | {error, term()}.
decode_module_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd) ->
case decode_bytes(BeamZstd) of
{ok, BeamZstdBytes} ->
{ok, #{
<<"module">> => Module,
<<"expected_current_md5">> => ExpectedMd5,
<<"target_md5">> => TargetMd5,
<<"beam_zstd">> => BeamZstdBytes
}};
{error, Reason} ->
{error, Reason}
end.
-spec json_to_event(term()) -> {ok, map()} | {error, term()}.
json_to_event(#{
<<"signer_key_id">> := SignerKeyId,
<<"bundle_sha256">> := BundleSha256,
<<"signature">> := Signature,
<<"bundle">> := Bundle
}) ->
decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle);
json_to_event(Other) ->
{error, {invalid_event_json, Other}}.
-spec decode_event_bytes(term(), term(), term(), term()) ->
{ok, map()} | {error, term()}.
decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) when
is_binary(SignerKeyId), is_binary(BundleSha256), is_binary(Signature), is_binary(Bundle)
->
case decode_bytes(BundleSha256) of
{ok, BundleSha256Bytes} ->
decode_event_signature(SignerKeyId, BundleSha256Bytes, Signature, Bundle);
{error, Reason} ->
{error, Reason}
end;
decode_event_bytes(SignerKeyId, BundleSha256, Signature, Bundle) ->
{error, {invalid_event_json, {SignerKeyId, BundleSha256, Signature, Bundle}}}.
-spec decode_event_signature(binary(), binary(), binary(), binary()) ->
{ok, map()} | {error, term()}.
decode_event_signature(SignerKeyId, BundleSha256, Signature, Bundle) ->
case decode_bytes(Signature) of
{ok, SignatureBytes} ->
decode_event_bundle(SignerKeyId, BundleSha256, SignatureBytes, Bundle);
{error, Reason} ->
{error, Reason}
end.
-spec decode_event_bundle(binary(), binary(), binary(), binary()) ->
{ok, map()} | {error, term()}.
decode_event_bundle(SignerKeyId, BundleSha256, Signature, Bundle) ->
case decode_bytes(Bundle) of
{ok, BundleBytes} ->
{ok, #{
signer_key_id => SignerKeyId,
bundle_sha256 => BundleSha256,
signature => Signature,
bundle => BundleBytes
}};
{error, Reason} ->
{error, Reason}
end.
-spec encode_bytes(binary()) -> binary().
encode_bytes(Binary) ->
base64:encode(Binary).
-spec decode_bytes(binary()) -> {ok, binary()} | {error, term()}.
decode_bytes(Encoded) when is_binary(Encoded) ->
try
{ok, base64:decode(Encoded)}
catch
Class:Reason -> {error, {invalid_base64_bytes, Encoded, Class, Reason}}
end.
-spec write_json_result({ok, map()} | {error, term()}) -> non_neg_integer().
write_json_result({ok, Term}) ->
write_stdout("~ts~n", [json:encode(Term)]),
0;
write_json_result({error, Reason}) ->
write_error(Reason).
-spec write_append_result({ok, binary()} | {error, term()}) -> non_neg_integer().
write_append_result({ok, EventId}) ->
write_stdout("appended hotpatch event ~ts~n", [gateway_hotpatch_loader:hex(EventId)]),
0;
write_append_result({error, Reason}) ->
write_error(Reason).
-spec write_error(term()) -> non_neg_integer().
write_error(Reason) ->
write_stderr("gateway hotpatch failed: ~0tp~n", [Reason]),
1.
-spec write_usage() -> ok.
write_usage() ->
write_stderr(
"usage: gateway_hotpatch_cli bundle BUILD_SHA module_a=/path/to/module_a.beam ...~n"
" gateway_hotpatch_cli sign SIGNER_KEY_ID PRIVATE_KEY_RAW_FILE bundle.json~n"
" gateway_hotpatch_cli append BUILD_SHA CREATED_BY signed-event.json~n",
[]
).
-spec write_stdout(io:format(), [term()]) -> ok.
write_stdout(Format, Args) ->
write_stream(standard_io, Format, Args).
-spec write_stderr(io:format(), [term()]) -> ok.
write_stderr(Format, Args) ->
write_stream(standard_error, Format, Args).
-spec write_stream(file:io_device() | standard_io | standard_error, io:format(), [term()]) ->
ok.
write_stream(Device, Format, Args) ->
Output = iolist_to_binary(io_lib:format(Format, Args)),
_ = file:write(Device, Output),
ok.
fluxer_gateway/src/gateway/gateway_hotpatch_loader.erl
-433
View File
@@ -1,433 +0,0 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_hotpatch_loader).
-typing([eqwalizer]).
-export([
apply_bundle/1,
current_md5/1,
beam_module/1,
beam_md5/1,
hex/1
]).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.
-type module_entry() :: map().
-type apply_result() :: {ok, applied | skipped, atom(), binary()} | {error, term()}.
-spec apply_bundle(map()) -> {ok, map()} | {error, term()}.
apply_bundle(Bundle) when is_map(Bundle) ->
case
{
entry_value(version, Bundle),
entry_value(build_sha, Bundle),
entry_value(modules, Bundle)
}
of
{1, BundleBuildSha, Modules} when is_list(Modules) ->
apply_versioned_bundle(BundleBuildSha, Modules);
_ ->
{error, invalid_bundle}
end;
apply_bundle(_Bundle) ->
{error, invalid_bundle}.
-spec apply_versioned_bundle(term(), [module_entry()]) -> {ok, map()} | {error, term()}.
apply_versioned_bundle(BundleBuildSha, Modules) ->
CurrentBuildSha = gateway_hotpatch_runtime:build_sha(),
case normalize_binary(BundleBuildSha) of
CurrentBuildSha -> apply_modules(Modules, #{applied => [], skipped => []});
Other -> {error, {build_sha_mismatch, Other, CurrentBuildSha}}
end.
-spec current_md5(atom()) -> {ok, binary()} | {error, term()}.
current_md5(Module) when is_atom(Module) ->
case hotpatch_loaded_md5(Module) of
{ok, Md5} -> {ok, Md5};
error -> current_md5_result(Module, current_md5_from_loaded_or_file(Module))
end.
-spec current_md5_result(atom(), {ok, binary()} | {error, term()}) ->
{ok, binary()} | {error, term()}.
current_md5_result(_Module, {ok, Md5}) ->
{ok, Md5};
current_md5_result(Module, {error, Reason}) ->
fallback_current_md5(Module, Reason).
-spec fallback_current_md5(atom(), term()) -> {ok, binary()} | {error, term()}.
fallback_current_md5(Module, Reason) ->
case hotpatch_loaded_md5(Module) of
{ok, Md5} -> {ok, Md5};
error -> {error, Reason}
end.
-spec current_md5_from_loaded_or_file(atom()) -> {ok, binary()} | {error, term()}.
current_md5_from_loaded_or_file(Module) ->
case code:get_object_code(Module) of
{Module, Beam, _File} when is_binary(Beam) -> beam_md5(Beam);
error -> current_md5_from_file(Module)
end.
-spec beam_module(binary()) -> {ok, atom()} | {error, term()}.
beam_module(Beam) when is_binary(Beam) ->
case beam_lib:info(Beam) of
Info when is_list(Info) -> beam_module_from_info(Info);
Other -> {error, {beam_info_failed, Other}}
end.
-spec beam_module_from_info(list()) -> {ok, atom()} | {error, term()}.
beam_module_from_info(Info) ->
case lists:keyfind(module, 1, Info) of
{module, Module} when is_atom(Module) -> {ok, Module};
_ -> {error, module_not_found}
end.
-spec beam_md5(binary()) -> {ok, binary()} | {error, term()}.
beam_md5(Beam) when is_binary(Beam) ->
case beam_lib:md5(Beam) of
{ok, {_Module, Md5}} when is_binary(Md5) -> {ok, Md5};
Error -> {error, {beam_md5_failed, Error}}
end.
-spec hex(binary()) -> binary().
hex(Binary) when is_binary(Binary) ->
iolist_to_binary([[hex_nibble(High), hex_nibble(Low)] || <<High:4, Low:4>> <= Binary]).
-spec apply_modules([module_entry()], map()) -> {ok, map()} | {error, term()}.
apply_modules([], Acc) ->
{ok, Acc#{module_count => length(maps:get(applied, Acc)) + length(maps:get(skipped, Acc))}};
apply_modules([Entry | Rest], Acc) ->
case apply_module(Entry) of
{ok, applied, Module, TargetMd5} ->
apply_modules(
Rest,
Acc#{applied => [{Module, hex(TargetMd5)} | maps:get(applied, Acc)]}
);
{ok, skipped, Module, TargetMd5} ->
apply_modules(
Rest,
Acc#{skipped => [{Module, hex(TargetMd5)} | maps:get(skipped, Acc)]}
);
{error, Reason} ->
{error, Reason}
end.
-spec apply_module(module_entry()) -> apply_result().
apply_module(Entry) ->
with_entry(Entry, fun apply_valid_entry/4).
-spec apply_valid_entry(atom(), binary(), binary(), binary()) -> apply_result().
apply_valid_entry(Module, ExpectedMd5, TargetMd5, Beam) ->
case current_md5(Module) of
{ok, TargetMd5} ->
{ok, skipped, Module, TargetMd5};
{ok, ExpectedMd5} ->
load_module(Module, Beam, TargetMd5);
{ok, CurrentMd5} ->
{error, {md5_mismatch, Module, hex(CurrentMd5), hex(ExpectedMd5)}};
{error, Reason} ->
{error, {current_md5_failed, Module, Reason}}
end.
-spec with_entry(module_entry(), fun((atom(), binary(), binary(), binary()) -> apply_result())) ->
apply_result().
with_entry(Entry, Fun) when is_map(Entry), is_function(Fun, 4) ->
case normalize_module(entry_value(module, Entry)) of
{ok, Module} ->
ExpectedMd5 = entry_value(expected_current_md5, Entry),
TargetMd5 = entry_value(target_md5, Entry),
BeamZstd = entry_value(beam_zstd, Entry),
with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun);
{error, Reason} ->
{error, Reason}
end;
with_entry(_Entry, _Fun) ->
{error, invalid_module_entry}.
-spec with_entry_beam(atom(), term(), term(), term(), fun(
(atom(), binary(), binary(), binary()) -> apply_result()
)) ->
apply_result().
with_entry_beam(Module, ExpectedMd5, TargetMd5, BeamZstd, Fun) when
is_binary(ExpectedMd5),
is_binary(TargetMd5),
byte_size(ExpectedMd5) =:= 16,
byte_size(TargetMd5) =:= 16,
is_binary(BeamZstd)
->
case decompress_beam(BeamZstd) of
{ok, Beam} -> validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun);
{error, Reason} -> {error, {beam_decompress_failed, Module, Reason}}
end;
with_entry_beam(Module, _ExpectedMd5, _TargetMd5, _BeamZstd, _Fun) ->
{error, {invalid_module_entry, Module}}.
-spec validate_beam(atom(), binary(), binary(), binary(), fun(
(atom(), binary(), binary(), binary()) -> apply_result()
)) ->
apply_result().
validate_beam(Module, ExpectedMd5, TargetMd5, Beam, Fun) ->
case beam_module(Beam) of
{ok, Module} -> validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun);
{ok, OtherModule} -> {error, {beam_module_mismatch, Module, OtherModule}};
{error, Reason} -> {error, {beam_module_failed, Module, Reason}}
end.
-spec validate_beam_md5(atom(), binary(), binary(), binary(), fun(
(atom(), binary(), binary(), binary()) -> apply_result()
)) ->
apply_result().
validate_beam_md5(Module, ExpectedMd5, TargetMd5, Beam, Fun) ->
case beam_md5(Beam) of
{ok, TargetMd5} ->
Fun(Module, ExpectedMd5, TargetMd5, Beam);
{ok, OtherMd5} ->
{error, {target_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}};
{error, Reason} ->
{error, {target_md5_failed, Module, Reason}}
end.
-spec load_module(atom(), binary(), binary()) -> apply_result().
load_module(Module, Beam, TargetMd5) ->
case code:soft_purge(Module) of
true -> load_purged_module(Module, Beam, TargetMd5);
false -> {error, {soft_purge_failed, Module}}
end.
-spec load_purged_module(atom(), binary(), binary()) -> apply_result().
load_purged_module(Module, Beam, TargetMd5) ->
case code:load_binary(Module, atom_to_list(Module) ++ ".beam", Beam) of
{module, Module} ->
put_hotpatch_loaded_md5(Module, TargetMd5),
verify_loaded_module(Module, TargetMd5);
{error, Reason} ->
{error, {load_binary_failed, Module, Reason}}
end.
-spec verify_loaded_module(atom(), binary()) -> apply_result().
verify_loaded_module(Module, TargetMd5) ->
case current_md5(Module) of
{ok, TargetMd5} ->
{ok, applied, Module, TargetMd5};
{ok, OtherMd5} ->
{error, {post_load_md5_mismatch, Module, hex(OtherMd5), hex(TargetMd5)}};
{error, Reason} ->
{error, {post_load_md5_failed, Module, Reason}}
end.
-spec current_md5_from_file(atom()) -> {ok, binary()} | {error, term()}.
current_md5_from_file(Module) ->
case code:which(Module) of
File when is_list(File) -> current_md5_from_path(Module, File);
preloaded -> {error, preloaded};
non_existing -> {error, not_loaded};
Other -> {error, {not_loadable, Other}}
end.
-spec current_md5_from_path(atom(), file:filename()) -> {ok, binary()} | {error, term()}.
current_md5_from_path(Module, File) ->
case beam_lib:md5(File) of
{ok, {Module, Md5}} when is_binary(Md5) -> {ok, Md5};
Error -> {error, {beam_file_md5_failed, Error}}
end.
-spec hotpatch_loaded_md5(atom()) -> {ok, binary()} | error.
hotpatch_loaded_md5(Module) ->
case persistent_term:get({?MODULE, loaded_md5, Module}, undefined) of
Md5 when is_binary(Md5), byte_size(Md5) =:= 16 -> {ok, Md5};
_ -> error
end.
-spec put_hotpatch_loaded_md5(atom(), binary()) -> ok.
put_hotpatch_loaded_md5(Module, Md5) ->
persistent_term:put({?MODULE, loaded_md5, Module}, Md5).
-spec decompress_beam(binary()) -> {ok, binary()} | {error, term()}.
decompress_beam(Compressed) ->
try erlang:apply(ezstd, decompress, [Compressed]) of
Beam when is_binary(Beam) -> {ok, Beam};
Beam when is_list(Beam) -> {ok, iolist_to_binary(Beam)};
{error, Reason} -> {error, Reason};
Other -> {error, Other}
catch
Class:Reason -> {error, {Class, Reason}}
end.
-spec normalize_module(term()) -> {ok, atom()} | {error, term()}.
normalize_module(Module) when is_atom(Module) ->
{ok, Module};
normalize_module(Module) when is_binary(Module) ->
try
{ok, binary_to_existing_atom(Module, utf8)}
catch
error:badarg -> {error, {unknown_module, Module}}
end;
normalize_module(Module) when is_list(Module) ->
normalize_module(type_conv:ensure_binary(Module));
normalize_module(Module) ->
{error, {invalid_module, Module}}.
-spec normalize_binary(term()) -> binary().
normalize_binary(Bin) when is_binary(Bin) -> Bin;
normalize_binary(List) when is_list(List) -> type_conv:ensure_binary(List);
normalize_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
normalize_binary(Other) -> term_to_binary(Other).
-spec entry_value(atom(), map()) -> term().
entry_value(Key, Entry) ->
maps:get(Key, Entry, maps:get(atom_to_binary(Key, utf8), Entry, undefined)).
-spec hex_nibble(0..15) -> integer().
hex_nibble(N) when N < 10 -> $0 + N;
hex_nibble(N) -> $a + (N - 10).
-ifdef(TEST).
beam_module_and_md5_test() ->
{module, ?MODULE} = code:ensure_loaded(?MODULE),
{?MODULE, Binary, _File} = code:get_object_code(?MODULE),
?assertEqual({ok, ?MODULE}, beam_module(Binary)),
{ok, Md5} = beam_md5(Binary),
?assertEqual(16, byte_size(Md5)).
apply_bundle_rejects_build_mismatch_test() ->
Bundle = #{version => 1, build_sha => <<"definitely-not-this-build">>, modules => []},
?assertMatch({error, {build_sha_mismatch, _, _}}, apply_bundle(Bundle)).
apply_bundle_accepts_binary_keys_test() ->
Bundle = #{
<<"version">> => 1,
<<"build_sha">> => gateway_hotpatch_runtime:build_sha(),
<<"modules">> => []
},
?assertEqual(
{ok, #{applied => [], skipped => [], module_count => 0}}, apply_bundle(Bundle)
).
apply_bundle_loads_new_beam_test() ->
Module = gateway_hotpatch_loader_test_target,
cleanup_test_module(Module),
try
Beam1 = compile_test_module(Module, 1),
Beam2 = compile_test_module(Module, 2),
{module, Module} = load_test_beam(Module, Beam1),
?assertEqual(1, erlang:apply(Module, version, [])),
{ok, ExpectedMd5} = current_md5(Module),
{Entry, TargetMd5} = beam_entry(Module, ExpectedMd5, Beam2),
Bundle = #{
version => 1,
build_sha => gateway_hotpatch_runtime:build_sha(),
modules => [Entry]
},
?assertEqual(
{ok, #{applied => [{Module, hex(TargetMd5)}], skipped => [], module_count => 1}},
apply_bundle(Bundle)
),
?assertEqual(2, erlang:apply(Module, version, []))
after
cleanup_test_module(Module)
end.
apply_bundle_leaves_prior_module_loaded_when_later_entry_fails_test() ->
ModuleA = gateway_hotpatch_loader_test_partial_a,
ModuleB = gateway_hotpatch_loader_test_partial_b,
cleanup_test_module(ModuleA),
cleanup_test_module(ModuleB),
try
BeamA1 = compile_test_module(ModuleA, 1),
BeamA2 = compile_test_module(ModuleA, 2),
BeamB1 = compile_test_module(ModuleB, 1),
BeamB2 = compile_test_module(ModuleB, 2),
{module, ModuleA} = load_test_beam(ModuleA, BeamA1),
{module, ModuleB} = load_test_beam(ModuleB, BeamB1),
{ok, Md5A1} = current_md5(ModuleA),
WrongExpectedMd5 = <<0:128>>,
{EntryA, _Md5A2} = beam_entry(ModuleA, Md5A1, BeamA2),
{EntryB, _Md5B2} = beam_entry(ModuleB, WrongExpectedMd5, BeamB2),
Bundle = #{
version => 1,
build_sha => gateway_hotpatch_runtime:build_sha(),
modules => [EntryA, EntryB]
},
?assertMatch({error, {md5_mismatch, ModuleB, _, _}}, apply_bundle(Bundle)),
?assertEqual(2, erlang:apply(ModuleA, version, [])),
?assertEqual(1, erlang:apply(ModuleB, version, []))
after
cleanup_test_module(ModuleA),
cleanup_test_module(ModuleB)
end.
hex_test() ->
?assertEqual(<<"0001020f10ff">>, hex(<<0, 1, 2, 15, 16, 255>>)).
compile_test_module(Module, Version) ->
Dir = test_compile_dir(),
Source = filename:join(Dir, atom_to_list(Module) ++ ".erl"),
SourceText = io_lib:format(
"-module(~p).~n-export([version/0]).~nversion() -> ~p.~n",
[Module, Version]
),
ok = file:write_file(Source, SourceText),
case compile:file(Source, [binary, return_errors, return_warnings]) of
{ok, Module, Beam} -> Beam;
{ok, Module, Beam, _Warnings} -> Beam;
Error -> erlang:error({test_module_compile_failed, Module, Error})
end.
test_compile_dir() ->
Root =
case os:getenv("TMPDIR") of
false -> "/tmp";
Value -> Value
end,
Dir = filename:join(Root, "fluxer_hotpatch_loader_tests"),
ok = filelib:ensure_dir(filename:join(Dir, "placeholder")),
Dir.
load_test_beam(Module, Beam) ->
cleanup_test_module(Module),
BeamPath = filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam"),
ok = file:write_file(BeamPath, Beam),
code:load_abs(filename:rootname(BeamPath)).
beam_entry(Module, ExpectedMd5, TargetBeam) ->
{ok, TargetMd5} = beam_md5(TargetBeam),
{ok, BeamZstd} = compress_test_beam(TargetBeam),
{
#{
module => Module,
expected_current_md5 => ExpectedMd5,
target_md5 => TargetMd5,
beam_zstd => BeamZstd
},
TargetMd5
}.
compress_test_beam(Beam) ->
try erlang:apply(ezstd, compress, [Beam, 3]) of
Compressed when is_binary(Compressed) -> {ok, Compressed};
{error, Reason} -> {error, Reason};
Other -> {error, Other}
catch
Class:Reason -> {error, {Class, Reason}}
end.
cleanup_test_module(Module) ->
_ = code:soft_purge(Module),
_ = code:purge(Module),
_ = code:delete(Module),
_ = code:purge(Module),
erase_hotpatch_loaded_md5(Module),
_ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".erl")),
_ = file:delete(filename:join(test_compile_dir(), atom_to_list(Module) ++ ".beam")),
ok.
-spec erase_hotpatch_loaded_md5(atom()) -> ok.
erase_hotpatch_loaded_md5(Module) ->
_ = persistent_term:erase({?MODULE, loaded_md5, Module}),
ok.
-endif.
fluxer_gateway/src/gateway/gateway_hotpatch_reconciler.erl
-319
View File
@@ -1,319 +0,0 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_hotpatch_reconciler).
-typing([eqwalizer]).
-behaviour(gen_server).
-export([start_link/0, is_ready/0, status/0, reconcile_async/0]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
-record(state, {
enabled = false :: boolean(),
build_sha = <<"dev">> :: binary(),
public_keys = #{} :: #{binary() => binary()},
applied_event_ids = [] :: [term()],
applied_count = 0 :: non_neg_integer(),
poll_interval_ms = 5000 :: pos_integer(),
last_error = undefined :: term()
}).
-type state() :: #state{}.
-define(SERVER, ?MODULE).
-define(POLL, poll).
-define(STARTUP_RECONCILE_RETRY_MS, 1000).
-spec start_link() -> gen_server:start_ret().
start_link() ->
gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
-spec is_ready() -> boolean().
is_ready() ->
gateway_hotpatch_runtime:is_ready().
-spec status() -> map().
status() ->
gateway_hotpatch_runtime:status().
-spec reconcile_async() -> ok.
reconcile_async() ->
gen_server:cast(?SERVER, reconcile).
-spec init([]) -> {ok, state()} | {stop, term()}.
init([]) ->
case gateway_hotpatch_runtime:is_enabled() of
false -> init_disabled();
true -> init_enabled()
end.
-spec handle_call(term(), {pid(), term()}, state()) -> {reply, term(), state()}.
handle_call(status, _From, State) ->
{reply, state_status(State, gateway_hotpatch_runtime:is_ready()), State};
handle_call(_Request, _From, State) ->
{reply, {error, unsupported_call}, State}.
-spec handle_cast(term(), state()) -> {noreply, state()}.
handle_cast(reconcile, State) ->
{noreply, reconcile_and_publish(State)};
handle_cast(_Request, State) ->
{noreply, State}.
-spec handle_info(term(), state()) -> {noreply, state()}.
handle_info(?POLL, #state{enabled = true} = State) ->
NewState = reconcile_and_publish(State),
schedule_poll(NewState),
{noreply, NewState};
handle_info(_Info, State) ->
{noreply, State}.
-spec terminate(term(), state()) -> ok.
terminate(_Reason, _State) ->
ok.
-spec code_change(term(), state(), term()) -> {ok, state()}.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-spec init_disabled() -> {ok, state()}.
init_disabled() ->
BuildSha = gateway_hotpatch_runtime:build_sha(),
State = #state{enabled = false, build_sha = BuildSha},
gateway_hotpatch_runtime:put_ready(true),
publish_status(State, true),
{ok, State}.
-spec init_enabled() -> {ok, state()} | {stop, term()}.
init_enabled() ->
gateway_hotpatch_runtime:put_ready(false),
BuildSha = gateway_hotpatch_runtime:build_sha(),
PollIntervalMs = runtime_pos_integer(hotpatch_poll_interval_ms, 5000),
PublicKeyConfig = gateway_hotpatch_runtime:get(hotpatch_public_keys, undefined),
case gateway_hotpatch_bundle:parse_public_keys(PublicKeyConfig) of
{ok, PublicKeys} when map_size(PublicKeys) > 0 ->
State0 = #state{
enabled = true,
build_sha = BuildSha,
public_keys = PublicKeys,
poll_interval_ms = max(1000, PollIntervalMs)
},
publish_status(State0, false),
init_enabled_connected(State0);
{ok, _Empty} ->
init_enabled_config_error(BuildSha, PollIntervalMs, missing_hotpatch_public_keys);
{error, Reason} ->
init_enabled_config_error(
BuildSha, PollIntervalMs, {invalid_hotpatch_public_keys, Reason}
)
end.
-spec init_enabled_config_error(binary(), pos_integer(), term()) -> {ok, state()}.
init_enabled_config_error(BuildSha, PollIntervalMs, Reason) ->
State = #state{
enabled = true,
build_sha = BuildSha,
poll_interval_ms = max(1000, PollIntervalMs),
last_error = Reason
},
gateway_hotpatch_runtime:put_ready(false),
publish_status(State, false),
logger:error("Gateway hotpatch configuration invalid: ~0tp", [Reason]),
schedule_poll(State),
{ok, State}.
-spec init_enabled_connected(state()) -> {ok, state()}.
init_enabled_connected(State0) ->
finish_enabled_startup(State0).
-spec finish_enabled_startup(state()) -> {ok, state()}.
finish_enabled_startup(State0) ->
TimeoutMs = runtime_pos_integer(hotpatch_startup_sync_timeout_ms, 30000),
DeadlineMs = erlang:monotonic_time(millisecond) + TimeoutMs,
finish_startup_reconcile(State0, startup_reconcile(State0, DeadlineMs)).
-spec finish_startup_reconcile(state(), {ok, state()} | {error, term()}) -> {ok, state()}.
finish_startup_reconcile(_State0, {ok, State}) ->
gateway_hotpatch_runtime:put_ready(true),
publish_status(State, true),
schedule_poll(State),
{ok, State};
finish_startup_reconcile(State0, {error, Reason}) ->
Error = {hotpatch_startup_sync_failed, Reason},
State = State0#state{last_error = Error},
gateway_hotpatch_runtime:put_ready(false),
publish_status(State, false),
logger:error("Gateway hotpatch startup sync failed; keeping node unready: ~0tp", [Error]),
schedule_poll(State),
{ok, State}.
-spec startup_reconcile(state(), integer()) -> {ok, state()} | {error, term()}.
startup_reconcile(State, DeadlineMs) ->
case reconcile_once(State) of
{ok, NewState} ->
{ok, NewState};
{error, Reason} ->
maybe_retry_startup_reconcile(State, DeadlineMs, Reason)
end.
-spec maybe_retry_startup_reconcile(state(), integer(), term()) ->
{ok, state()} | {error, term()}.
maybe_retry_startup_reconcile(State, DeadlineMs, Reason) ->
case erlang:monotonic_time(millisecond) >= DeadlineMs of
true ->
{error, Reason};
false ->
retry_startup_reconcile(State#state{last_error = Reason}, DeadlineMs, Reason)
end.
-spec retry_startup_reconcile(state(), integer(), term()) -> {ok, state()} | {error, term()}.
retry_startup_reconcile(State, DeadlineMs, Reason) ->
case gateway_retry_timer:wait_until(?STARTUP_RECONCILE_RETRY_MS, DeadlineMs) of
ok -> startup_reconcile(State, DeadlineMs);
expired -> {error, Reason};
{error, _InvalidDelay} -> {error, Reason}
end.
-spec reconcile_and_publish(state()) -> state().
reconcile_and_publish(#state{enabled = false} = State) ->
gateway_hotpatch_runtime:put_ready(true),
publish_status(State, true),
State;
reconcile_and_publish(State) ->
case reconcile_once(State) of
{ok, NewState} ->
gateway_hotpatch_runtime:put_ready(true),
publish_status(NewState, true),
NewState;
{error, Reason} ->
publish_reconcile_error(State, Reason)
end.
-spec reconcile_once(state()) -> {ok, state()} | {error, term()}.
reconcile_once(#state{build_sha = BuildSha} = State) ->
case gateway_hotpatch_store:connect() of
ok -> fetch_and_apply_events(BuildSha, State);
{error, Reason} -> {error, {store_connect_failed, Reason}}
end.
-spec fetch_and_apply_events(binary(), state()) -> {ok, state()} | {error, term()}.
fetch_and_apply_events(BuildSha, State) ->
case gateway_hotpatch_store:fetch_events(BuildSha) of
{ok, Events} -> apply_events(Events, State);
{error, Reason} -> {error, {fetch_events_failed, Reason}}
end.
-spec publish_reconcile_error(state(), term()) -> state().
publish_reconcile_error(State, {fetch_events_failed, _Reason} = Error) ->
ErrorState = State#state{last_error = Error},
Ready = gateway_hotpatch_runtime:is_ready(),
gateway_hotpatch_runtime:put_ready(Ready),
publish_status(ErrorState, Ready),
logger:warning("Gateway hotpatch fetch failed; keeping current readiness: ~0tp", [Error]),
ErrorState;
publish_reconcile_error(State, Reason) ->
ErrorState = State#state{last_error = Reason},
gateway_hotpatch_runtime:put_ready(false),
publish_status(ErrorState, false),
logger:error("Gateway hotpatch reconciliation failed: ~0tp", [Reason]),
ErrorState.
-spec apply_events([map()], state()) -> {ok, state()} | {error, term()}.
apply_events([], State) ->
{ok, State#state{last_error = undefined}};
apply_events([Event | Rest], State) ->
EventId = maps:get(event_id, Event, undefined),
case lists:member(EventId, State#state.applied_event_ids) of
true ->
apply_events(Rest, State);
false ->
apply_new_event(EventId, Event, Rest, State)
end.
-spec apply_new_event(term(), map(), [map()], state()) -> {ok, state()} | {error, term()}.
apply_new_event(EventId, Event, Rest, State) ->
case apply_event(Event, State) of
{ok, NewState} -> apply_events(Rest, NewState);
{error, Reason} -> {error, {event_apply_failed, EventId, Reason}}
end.
-spec apply_event(map(), state()) -> {ok, state()} | {error, term()}.
apply_event(Event, #state{public_keys = PublicKeys, build_sha = BuildSha} = State) ->
EventId = maps:get(event_id, Event, undefined),
BundleHash = maps:get(bundle_sha256, Event, <<>>),
_ = code:ensure_loaded(gateway_hotpatch_loader),
case gateway_hotpatch_bundle:decode_signed_event(Event, PublicKeys) of
{ok, Bundle} ->
apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State);
{error, Reason} ->
audit_event(BuildSha, EventId, #{bundle_sha256 => BundleHash}, {error, Reason}),
{error, Reason}
end.
-spec apply_decoded_event(binary(), term(), binary(), map(), state()) ->
{ok, state()} | {error, term()}.
apply_decoded_event(BuildSha, EventId, BundleHash, Bundle, State) ->
Summary0 = #{bundle_sha256 => BundleHash},
case gateway_hotpatch_loader:apply_bundle(Bundle) of
{ok, Summary} ->
handle_applied_event(BuildSha, EventId, Summary0, Summary, State);
{error, Reason} ->
audit_event(BuildSha, EventId, Summary0, {error, Reason}),
{error, Reason}
end.
-spec handle_applied_event(binary(), term(), map(), map(), state()) -> {ok, state()}.
handle_applied_event(BuildSha, EventId, Summary0, Summary, State) ->
AuditSummary = maps:merge(Summary0, Summary),
audit_event(BuildSha, EventId, AuditSummary, ok),
logger:notice("Applied gateway hotpatch event ~0tp summary=~0tp", [EventId, AuditSummary]),
{ok, State#state{
applied_event_ids = [EventId | State#state.applied_event_ids],
applied_count = State#state.applied_count + 1,
last_error = undefined
}}.
-spec audit_event(binary(), term(), map(), ok | {error, term()}) -> ok.
audit_event(BuildSha, EventId, Summary, Result) when is_binary(EventId) ->
case
gateway_hotpatch_store:audit_applied(
BuildSha, gateway_hotpatch_runtime:node_name(), EventId, Summary, Result
)
of
ok ->
ok;
{error, Reason} ->
logger:warning("Gateway hotpatch audit write failed: ~0tp", [Reason]),
ok
end;
audit_event(_BuildSha, _EventId, _Summary, _Result) ->
ok.
-spec publish_status(state(), boolean()) -> ok.
publish_status(State, Ready) ->
gateway_hotpatch_runtime:put_status(state_status(State, Ready)).
-spec state_status(state(), boolean()) -> map().
state_status(State, Ready) ->
#{
enabled => State#state.enabled,
ready => Ready,
build_sha => State#state.build_sha,
applied_count => State#state.applied_count,
applied_event_count => length(State#state.applied_event_ids),
last_error => State#state.last_error
}.
-spec schedule_poll(state()) -> ok.
schedule_poll(#state{poll_interval_ms = PollIntervalMs}) ->
erlang:send_after(PollIntervalMs, self(), ?POLL),
ok.
-spec runtime_pos_integer(atom(), pos_integer()) -> pos_integer().
runtime_pos_integer(Key, Default) ->
normalize_pos_integer(gateway_hotpatch_runtime:get(Key, Default), Default).
-spec normalize_pos_integer(term(), pos_integer()) -> pos_integer().
normalize_pos_integer(Value, _Default) when is_integer(Value), Value > 0 ->
Value;
normalize_pos_integer(_Value, Default) ->
Default.
fluxer_gateway/src/gateway/gateway_hotpatch_runtime.erl
-78
View File
@@ -1,78 +0,0 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_hotpatch_runtime).
-typing([eqwalizer]).
-export([
build_sha/0,
node_name/0,
is_enabled/0,
get/2,
put_ready/1,
is_ready/0,
put_status/1,
status/0
]).
-define(READY_KEY, {gateway_hotpatch, ready}).
-define(STATUS_KEY, {gateway_hotpatch, status}).
-spec build_sha() -> binary().
build_sha() ->
case os:getenv("BUILD_SHA") of
false -> build_version();
"" -> build_version();
Value -> type_conv:ensure_binary(Value, <<"dev">>)
end.
-spec build_version() -> binary().
build_version() ->
case os:getenv("BUILD_VERSION") of
false -> <<"dev">>;
"" -> <<"dev">>;
Value -> type_conv:ensure_binary(Value, <<"dev">>)
end.
-spec node_name() -> binary().
node_name() ->
atom_to_binary(node(), utf8).
-spec is_enabled() -> boolean().
is_enabled() ->
case fluxer_gateway_env:get(hotpatch_enabled) of
true -> true;
_ -> false
end.
-spec get(atom(), term()) -> term().
get(Key, Default) ->
case fluxer_gateway_env:get(Key) of
undefined -> Default;
Value -> Value
end.
-spec put_ready(boolean()) -> ok.
put_ready(Ready) when is_boolean(Ready) ->
persistent_term:put(?READY_KEY, Ready),
ok.
-spec is_ready() -> boolean().
is_ready() ->
case persistent_term:get(?READY_KEY, true) of
false -> false;
_ -> true
end.
-spec put_status(map()) -> ok.
put_status(Status) ->
persistent_term:put(?STATUS_KEY, Status),
ok.
-spec status() -> map().
status() ->
persistent_term:get(?STATUS_KEY, #{
enabled => false,
ready => true,
build_sha => build_sha(),
applied_count => 0
}).
fluxer_gateway/src/gateway/gateway_hotpatch_store.erl
-277
View File
@@ -1,277 +0,0 @@
%% SPDX-License-Identifier: AGPL-3.0-or-later
-module(gateway_hotpatch_store).
-typing([eqwalizer]).
-export([
connect/0,
fetch_events/1,
append_event/5,
append_event/6,
audit_applied/5,
normalize_event_rows/1
]).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.
-define(STMT_FETCH_EVENTS, gateway_hotpatch_fetch_events_by_build).
-define(STMT_APPEND_EVENT, gateway_hotpatch_append_event).
-define(STMT_AUDIT_APPLIED, gateway_hotpatch_audit_applied).
-spec connect() -> ok | {error, term()}.
connect() ->
case gateway_hotpatch_runtime:get(hotpatch_cassandra_hosts, undefined) of
Hosts when is_list(Hosts), Hosts =/= [] -> connect_hosts(Hosts);
Hosts when is_binary(Hosts), byte_size(Hosts) > 0 ->
start_erlcass(binary_to_list(Hosts));
_ ->
{error, missing_hotpatch_cassandra_hosts}
end.
-spec connect_hosts(term()) -> ok | {error, term()}.
connect_hosts(Hosts) ->
case type_conv:to_list(Hosts) of
String when is_list(String), String =/= [] -> start_erlcass(String);
_ -> {error, missing_hotpatch_cassandra_hosts}
end.
-spec fetch_events(binary()) -> {ok, [map()]} | {error, term()}.
fetch_events(BuildSha) when is_binary(BuildSha) ->
case erlcass:execute(?STMT_FETCH_EVENTS, [BuildSha]) of
{ok, _Columns, Rows} -> {ok, normalize_event_rows(Rows)};
ok -> {ok, []};
{error, Reason} -> {error, Reason}
end.
-spec append_event(binary(), binary(), binary(), binary(), binary()) ->
{ok, binary()} | {error, term()}.
append_event(CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when
is_binary(CreatedBy),
is_binary(SignerKeyId),
is_binary(Signature),
is_binary(BundleSha256),
is_binary(Bundle)
->
BuildSha = gateway_hotpatch_runtime:build_sha(),
append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle).
-spec append_event(binary(), binary(), binary(), binary(), binary(), binary()) ->
{ok, binary()} | {error, term()}.
append_event(BuildSha, CreatedBy, SignerKeyId, Signature, BundleSha256, Bundle) when
is_binary(BuildSha),
is_binary(CreatedBy),
is_binary(SignerKeyId),
is_binary(Signature),
is_binary(BundleSha256),
is_binary(Bundle)
->
case erlcass_uuid:gen_time() of
{ok, EventId} ->
Params = append_event_params(
BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle
),
execute_append_event(EventId, Params);
{error, Reason} ->
{error, {event_id_failed, Reason}}
end.
-spec append_event_params(
binary(), binary(), binary(), binary(), binary(), binary(), binary()
) -> list().
append_event_params(BuildSha, EventId, CreatedBy, SignerKeyId, BundleSha256, Signature, Bundle) ->
[
BuildSha,
EventId,
1,
<<"beam_bundle">>,
CreatedBy,
SignerKeyId,
BundleSha256,
Signature,
Bundle
].
-spec execute_append_event(binary(), list()) -> {ok, binary()} | {error, term()}.
execute_append_event(EventId, Params) ->
case erlcass:execute(?STMT_APPEND_EVENT, Params) of
ok -> {ok, EventId};
{ok, _Columns, _Rows} -> {ok, EventId};
{error, Reason} -> {error, Reason}
end.
-spec audit_applied(binary(), binary(), binary(), map(), ok | {error, term()}) ->
ok | {error, term()}.
audit_applied(BuildSha, NodeName, EventId, Summary, Result) when
is_binary(BuildSha), is_binary(NodeName), is_binary(EventId), is_map(Summary)
->
ModuleCount = maps:get(module_count, Summary, 0),
BundleSha256 = maps:get(bundle_sha256, Summary, <<>>),
Status = audit_status(Result),
Error = audit_error(Result),
Params = [BuildSha, NodeName, EventId, ModuleCount, BundleSha256, Status, Error],
case erlcass:execute(?STMT_AUDIT_APPLIED, Params) of
ok -> ok;
{ok, _Columns, _Rows} -> ok;
{error, Reason} -> {error, Reason}
end.
-spec normalize_event_rows([list()]) -> [map()].
normalize_event_rows(Rows) when is_list(Rows) ->
lists:filtermap(fun normalize_event_row/1, Rows).
-spec start_erlcass(string()) -> ok | {error, term()}.
start_erlcass(Hosts) ->
Keyspace = gateway_hotpatch_runtime:get(hotpatch_cassandra_keyspace, <<"fluxer">>),
Port = gateway_hotpatch_runtime:get(hotpatch_cassandra_port, 9042),
Options0 = [
{contact_points, type_conv:ensure_binary(Hosts)},
{port, Port},
{latency_aware_routing, true},
{token_aware_routing, true},
{tcp_nodelay, true},
{tcp_keepalive, {true, 60}},
{connect_timeout, 5000},
{request_timeout, 5000},
{retry_policy, {default, true}},
{default_consistency_level, 6}
],
case maybe_credentials(Options0) of
{ok, Options} -> start_erlcass_with_options(Keyspace, Options);
{error, Reason} -> {error, Reason}
end.
-spec start_erlcass_with_options(binary(), list()) -> ok | {error, term()}.
start_erlcass_with_options(Keyspace, Options) ->
application:set_env(erlcass, keyspace, Keyspace),
application:set_env(erlcass, cluster_options, Options),
application:set_env(erlcass, log_level, 2),
case application:ensure_all_started(erlcass) of
{ok, _Apps} -> prepare_statements();
{error, {erlcass, {already_started, erlcass}}} -> prepare_statements();
{error, Reason} -> {error, {erlcass_start_failed, Reason}}
end.
-spec prepare_statements() -> ok | {error, term()}.
prepare_statements() ->
Statements = [
{?STMT_FETCH_EVENTS, <<
"SELECT event_id, schema_version, kind, created_by, signer_key_id, "
"bundle_sha256, signature, bundle "
"FROM gateway_hotpatch_events_by_build WHERE build_sha = ?"
>>},
{?STMT_APPEND_EVENT, <<
"INSERT INTO gateway_hotpatch_events_by_build "
"(build_sha, event_id, schema_version, kind, created_at, created_by, "
"signer_key_id, bundle_sha256, signature, bundle) "
"VALUES (?, ?, ?, ?, toTimestamp(now()), ?, ?, ?, ?, ?)"
>>},
{?STMT_AUDIT_APPLIED, <<
"INSERT INTO gateway_hotpatch_applied_by_node "
"(build_sha, node_name, event_id, applied_at, module_count, bundle_sha256, "
"status, error) "
"VALUES (?, ?, ?, toTimestamp(now()), ?, ?, ?, ?)"
>>}
],
prepare_statements(Statements).
-spec prepare_statements([{atom(), binary()}]) -> ok | {error, term()}.
prepare_statements([]) ->
ok;
prepare_statements([{Id, Query} | Rest]) ->
case prepare_statement(Id, Query) of
ok -> prepare_statements(Rest);
{error, Reason} -> {error, Reason}
end.
-spec prepare_statement(atom(), binary()) -> ok | {error, term()}.
prepare_statement(Id, Query) ->
case erlcass:add_prepare_statement(Id, Query) of
ok -> ok;
{error, already_exist} -> ok;
{error, Reason} -> {error, {prepare_failed, Id, Reason}}
end.
-spec maybe_credentials(list()) -> {ok, list()} | {error, term()}.
maybe_credentials(Options) ->
Username = gateway_hotpatch_runtime:get(hotpatch_cassandra_username, undefined),
Password = gateway_hotpatch_runtime:get(hotpatch_cassandra_password, undefined),
case {Username, Password} of
{U, P} when is_binary(U), byte_size(U) > 0, is_binary(P), byte_size(P) > 0 ->
{ok, [{credentials, {U, P}} | Options]};
_ ->
{error, missing_hotpatch_cassandra_credentials}
end.
-spec normalize_event_row(list()) -> {true, map()} | false.
normalize_event_row([
EventId,
SchemaVersion,
Kind,
CreatedBy,
SignerKeyId,
BundleSha256,
Signature,
Bundle
]) when
is_binary(EventId),
is_binary(SignerKeyId),
is_binary(BundleSha256),
is_binary(Signature),
is_binary(Bundle)
->
{true, #{
event_id => EventId,
schema_version => SchemaVersion,
kind => Kind,
created_by => CreatedBy,
signer_key_id => SignerKeyId,
bundle_sha256 => BundleSha256,
signature => Signature,
bundle => Bundle
}};
normalize_event_row(_Row) ->
false.
-spec audit_status(ok | {error, term()}) -> binary().
audit_status(ok) -> <<"ok">>;
audit_status({error, _Reason}) -> <<"error">>.
-spec audit_error(ok | {error, term()}) -> binary().
audit_error(ok) ->
<<>>;
audit_error({error, Reason}) ->
iolist_to_binary(io_lib:format("~0tp", [Reason])).
-ifdef(TEST).
normalize_event_rows_test() ->
EventId = <<1:128>>,
Row = [
EventId,
1,
<<"beam_bundle">>,
<<"hampus">>,
<<"ops">>,
<<2:256>>,
<<3:512>>,
<<"bundle">>
],
?assertEqual(
[
#{
event_id => EventId,
schema_version => 1,
kind => <<"beam_bundle">>,
created_by => <<"hampus">>,
signer_key_id => <<"ops">>,
bundle_sha256 => <<2:256>>,
signature => <<3:512>>,
bundle => <<"bundle">>
}
],
normalize_event_rows([Row, [invalid]])
).
-endif.
fluxer_gateway/src/gateway/gateway_node_router.erl
+1 -1
View File
@@ -98,7 +98,7 @@ fallback_owner_for_role(Role) ->
-spec is_ready() -> boolean().
is_ready() ->
not is_draining() andalso gateway_hotpatch_reconciler:is_ready().
not is_draining().
-spec is_draining() -> boolean().
is_draining() ->
fluxer_gateway/src/gateway/metrics_handler.erl
-32
View File
@@ -23,7 +23,6 @@ render_metrics() ->
render_cluster_counters(),
render_process_counts(),
render_push_dispatcher_stats(),
render_hotpatch_metrics(),
render_vm_metrics()
].
@@ -256,33 +255,6 @@ render_push_dispatcher_stats() ->
]
end.
-spec render_hotpatch_metrics() -> iolist().
render_hotpatch_metrics() ->
Status = safe_apply_map(fun gateway_hotpatch_reconciler:status/0),
Ready = maps:get(ready, Status, true),
Enabled = maps:get(enabled, Status, false),
AppliedCount = maps:get(applied_count, Status, 0),
[
format_metric(
<<"fluxer_gateway_hotpatch_enabled">>,
<<"gauge">>,
<<"Gateway hotpatch reconciler enabled">>,
bool_metric(Enabled)
),
format_metric(
<<"fluxer_gateway_hotpatch_ready">>,
<<"gauge">>,
<<"Gateway hotpatch reconciliation readiness">>,
bool_metric(Ready)
),
format_metric(
<<"fluxer_gateway_hotpatch_applied_events_total">>,
<<"counter">>,
<<"Gateway hotpatch events applied on this node">>,
integer_to_binary(AppliedCount)
)
].
-spec render_vm_metrics() -> iolist().
render_vm_metrics() ->
Memory = safe_apply_list(fun erlang:memory/0),
@@ -376,7 +348,3 @@ format_labeled_series(Name, Type, Help, LabelValues) ->
|| {Label, Value} <- LabelValues
]
].
-spec bool_metric(term()) -> binary().
bool_metric(true) -> <<"1">>;
bool_metric(_) -> <<"0">>.
fluxer_gateway/src/guild/guild_member_list_sync_batch.erl
+1 -1
View File
@@ -159,7 +159,7 @@ flush_pending_syncs_clears_empty_batch_test() ->
State = #{?SYNC_BATCH_STATE_KEY => #{pending_list_ids => #{}}},
?assertEqual(#{}, flush_pending_syncs(State)).
queue_list_sync_drains_pre_hotpatch_pending_batch_test() ->
queue_list_sync_drains_preexisting_pending_batch_test() ->
Ref = erlang:send_after(60000, self(), ?FLUSH_SYNC_BATCH_MSG),
State = #{
?SYNC_BATCH_STATE_KEY => #{
fluxer_svc/src/postgres.rs
+31 -2
View File
@@ -11,6 +11,9 @@ use std::str::FromStr;
use tokio_postgres::{Config as PgConfig, Row, config::SslMode, types::ToSql};
use tokio_postgres_rustls::MakeRustlsConnect;
const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE: i32 = 0x4658_4b56;
const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT: &str = "120s";
#[derive(Clone, Debug)]
pub struct PostgresConfig {
pub url: Option<String>,
@@ -134,8 +137,30 @@ pub async fn ensure_kv_schema(pool: &Pool, kv_table: &str) -> anyhow::Result<()>
let messages_message_index = quote_identifier(&format!("{kv_table}_messages_message_idx"))?;
let message_reactions_message_index =
quote_identifier(&format!("{kv_table}_message_reactions_message_idx"))?;
let client = pool.get().await?;
client
let mut client = pool.get().await?;
let transaction = client
.transaction()
.await
.context("failed to begin Postgres KV schema transaction")?;
transaction
.query_one(
"SELECT set_config('statement_timeout', $1, true)",
&[&POSTGRES_KV_SCHEMA_LOCK_TIMEOUT],
)
.await
.context("failed to configure Postgres KV schema lock timeout")?;
transaction
.query_one(
"SELECT pg_advisory_xact_lock($1, hashtext($2))",
&[&POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, &kv_table],
)
.await
.context("failed to acquire Postgres KV schema lock")?;
transaction
.query_one("SELECT set_config('statement_timeout', '0', true)", &[])
.await
.context("failed to clear Postgres KV schema lock timeout")?;
transaction
.batch_execute(&format!(
r#"
CREATE TABLE IF NOT EXISTS {table} (
@@ -162,6 +187,10 @@ DROP INDEX IF EXISTS {old_partition_index};
))
.await
.context("failed to ensure Postgres KV schema")?;
transaction
.commit()
.await
.context("failed to commit Postgres KV schema transaction")?;
Ok(())
}
package.json
+4 -5
View File
@@ -13,11 +13,10 @@
"dev:desktop:canary": "cargo run -p fluxer-dev -- desktop canary",
"dev:desktop:package": "cargo run -p fluxer-dev -- desktop package",
"dev:desktop:run": "cargo run -p fluxer-dev -- desktop run",
"dev:infra:down": "docker compose -f .devcontainer/docker-compose.yml down",
"dev:infra:restart": "pnpm dev:infra:down && pnpm dev:infra:up",
"dev:infra:up": "pnpm dev:infra:up:lite",
"dev:infra:up:full": "docker compose -f .devcontainer/docker-compose.yml up -d postgres cassandra valkey nats livekit elasticsearch meilisearch mailpit",
"dev:infra:up:lite": "docker compose -f .devcontainer/docker-compose.yml up -d postgres valkey nats livekit meilisearch mailpit",
"dev:infra:restart": "pnpm dev:infra:stop && pnpm dev:infra:start",
"dev:infra:start": "cargo run -p fluxer-dev -- infra start",
"dev:infra:status": "cargo run -p fluxer-dev -- infra status",
"dev:infra:stop": "cargo run -p fluxer-dev -- infra stop",
"dev:services": "cargo run -p fluxer-dev -- rust-services",
"dev:tunnel": "cargo run -p fluxer-dev -- dev --cloudflare-tunnel",
"dev:tunnel:configure": "cargo run -p fluxer-dev -- tunnel configure",
packages/config/src/ConfigLoader.ts
+1
View File
@@ -195,6 +195,7 @@ function defaultConfig(): MasterConfig {
api_key: '',
api_secret: '',
url: '',
internal_url: '',
webhook_url: '',
},
search: {
packages/config/src/MasterConfig.ts
+1
View File
@@ -243,6 +243,7 @@ export interface MasterConfig {
api_key: string;
api_secret: string;
url: string;
internal_url: string;
webhook_url: string;
default_region?: {
id: string;
packages/config/src/config_loader/EnvironmentOverrides.ts
+1
View File
@@ -266,6 +266,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_LIVEKIT_API_KEY: {path: ['integrations', 'voice', 'api_key']},
FLUXER_LIVEKIT_API_SECRET: {path: ['integrations', 'voice', 'api_secret']},
FLUXER_LIVEKIT_URL: {path: ['integrations', 'voice', 'url']},
FLUXER_LIVEKIT_INTERNAL_URL: {path: ['integrations', 'voice', 'internal_url']},
FLUXER_LIVEKIT_WEBHOOK_URL: {path: ['integrations', 'voice', 'webhook_url']},
FLUXER_LIVEKIT_DEFAULT_REGION: {path: ['integrations', 'voice', 'default_region'], parse: parseEnvValue},
FLUXER_SEARCH_ENGINE: {path: ['integrations', 'search', 'engine']},
pnpm-lock.yaml
Generated
+5 -5
View File
@@ -7701,8 +7701,8 @@ packages:
resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==}
engines: {node: '>=16'}
caniuse-lite@1.0.30001774:
resolution: {integrity: sha512-DDdwPGz99nmIEv216hKSgLD+D4ikHQHjBC/seF98N9CPqRX4M5mSxT9eTV6oyisnJcuzxtZy4n17yKKQYmYQOA==}
caniuse-lite@1.0.30001810:
resolution: {integrity: sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==}
cassandra-driver@4.8.0:
resolution: {integrity: sha512-HritfMGq9V7SuESeSodHvArs0mLuMk7uh+7hQK2lqdvXrvm50aWxb4RPxkK3mPDdsgHjJ427xNRFITMH2ei+Sw==}
@@ -17551,7 +17551,7 @@ snapshots:
autoprefixer@10.4.24(postcss@8.5.6):
dependencies:
browserslist: 4.28.1
caniuse-lite: 1.0.30001774
caniuse-lite: 1.0.30001810
fraction.js: 5.3.4
picocolors: 1.1.1
postcss: 8.5.6
@@ -17666,7 +17666,7 @@ snapshots:
browserslist@4.28.1:
dependencies:
baseline-browser-mapping: 2.10.0
caniuse-lite: 1.0.30001774
caniuse-lite: 1.0.30001810
electron-to-chromium: 1.5.302
node-releases: 2.0.27
update-browserslist-db: 1.2.3(browserslist@4.28.1)
@@ -17761,7 +17761,7 @@ snapshots:
camelcase@8.0.0: {}
caniuse-lite@1.0.30001774: {}
caniuse-lite@1.0.30001810: {}
cassandra-driver@4.8.0:
dependencies:
tools/ci/src/app_dev_server.rs
+123 -51
View File
@@ -4,9 +4,11 @@ use crate::app_wasm::resolve_app_dir;
use anyhow::{Context, Result, bail, ensure};
use clap::Args;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsStr;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
@@ -23,11 +25,51 @@ pub struct AppDevServerArgs {
app_dir: Option<PathBuf>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(untagged)]
enum InputFingerprint {
Sha256 {
sha256: String,
#[serde(skip)]
legacy_timestamp_ms: Option<f64>,
},
LegacyTimestamp(f64),
}
impl InputFingerprint {
fn matches(&self, current: &Self) -> bool {
match (self, current) {
(
Self::Sha256 { sha256, .. },
Self::Sha256 {
sha256: current, ..
},
) => sha256 == current,
(
Self::LegacyTimestamp(timestamp),
Self::Sha256 {
legacy_timestamp_ms: Some(current),
..
},
) => timestamp == current,
(Self::LegacyTimestamp(timestamp), Self::LegacyTimestamp(current)) => {
timestamp == current
}
_ => false,
}
}
fn is_legacy(&self) -> bool {
matches!(self, Self::LegacyTimestamp(_))
}
}
type StepInputs = BTreeMap<String, InputFingerprint>;
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct StepMetadata {
last_run: f64,
inputs: BTreeMap<String, f64>,
inputs: StepInputs,
}
type Metadata = BTreeMap<String, StepMetadata>;
@@ -103,20 +145,14 @@ impl AppDevServer {
)
.await?;
if env_truthy("FLUXER_APP_SKIP_I18N_COMPILE") {
eprintln!("Skipping pnpm lingui:compile because FLUXER_APP_SKIP_I18N_COMPILE is set.");
} else {
self.run_cached_step(
"lingui",
gather_lingui_inputs,
"pnpm lingui:compile",
|server, shutdown| {
Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown))
},
&mut shutdown_rx,
)
.await?;
}
self.run_cached_step(
"lingui",
gather_lingui_inputs,
"pnpm lingui:compile",
|server, shutdown| Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown)),
&mut shutdown_rx,
)
.await?;
if *shutdown_rx.borrow() {
return Ok(());
@@ -175,35 +211,50 @@ impl AppDevServer {
shutdown: &mut watch::Receiver<bool>,
) -> Result<()>
where
G: Fn(&Path) -> Result<BTreeMap<String, f64>>,
G: Fn(&Path) -> Result<StepInputs>,
E: for<'a> FnOnce(
&'a AppDevServer,
&'a mut watch::Receiver<bool>,
)
-> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>>,
{
if *shutdown.borrow() {
return Ok(());
}
let inputs = gather_inputs(&self.project_root)?;
if !self.should_run_step(step_name, &inputs) {
let needs_upgrade = self
.metadata
.get(step_name)
.is_some_and(|entry| entry.inputs.values().any(InputFingerprint::is_legacy));
if needs_upgrade {
self.metadata
.insert(step_name.to_string(), StepMetadata { inputs });
self.save_metadata()?;
}
println!("Skipping {label} (no changes detected)");
return Ok(());
}
execute(self, shutdown).await?;
self.metadata.insert(
step_name.to_string(),
StepMetadata {
last_run: timestamp_ms(SystemTime::now())?,
inputs,
},
);
if *shutdown.borrow() {
return Ok(());
}
self.metadata
.insert(step_name.to_string(), StepMetadata { inputs });
self.save_metadata()
}
fn should_run_step(&self, step_name: &str, inputs: &BTreeMap<String, f64>) -> bool {
fn should_run_step(&self, step_name: &str, inputs: &StepInputs) -> bool {
let Some(entry) = self.metadata.get(step_name) else {
return true;
};
&entry.inputs != inputs
entry.inputs.len() != inputs.len()
|| entry.inputs.iter().any(|(path, cached)| {
inputs
.get(path)
.is_none_or(|current| !cached.matches(current))
})
}
async fn run_command(
@@ -286,7 +337,7 @@ impl AppDevServer {
}
}
fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap<String, f64>> {
fn collect_file_digests(project_root: &Path, paths: &[PathBuf]) -> Result<StepInputs> {
let mut result = BTreeMap::new();
for rel_path in paths {
let absolute_path = project_root.join(rel_path);
@@ -297,16 +348,16 @@ fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap
"Expected {} to be a file when collecting dev server cache inputs.",
rel_path.display()
);
result.insert(rel_path_key(rel_path), timestamp_ms(metadata.modified()?)?);
result.insert(rel_path_key(rel_path), fingerprint_file(&absolute_path)?);
}
Ok(result)
}
fn collect_directory_stats<P>(
fn collect_directory_digests<P>(
project_root: &Path,
root_rel: &Path,
predicate: P,
) -> Result<BTreeMap<String, f64>>
) -> Result<StepInputs>
where
P: Fn(&str) -> bool,
{
@@ -335,20 +386,47 @@ where
if !predicate(&key) {
continue;
}
result.insert(key, timestamp_ms(entry.metadata()?.modified()?)?);
result.insert(key, fingerprint_file(entry.path())?);
}
Ok(result)
}
fn hash_file(path: &Path) -> Result<String> {
let mut file = fs::File::open(path)
.with_context(|| format!("Failed to open {} for hashing", path.display()))?;
let mut hasher = Sha256::new();
let mut buffer = [0_u8; 64 * 1024];
loop {
let bytes_read = file
.read(&mut buffer)
.with_context(|| format!("Failed to read {} for hashing", path.display()))?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
Ok(hex::encode(hasher.finalize()))
}
fn fingerprint_file(path: &Path) -> Result<InputFingerprint> {
let sha256 = hash_file(path)?;
let metadata = fs::metadata(path)
.with_context(|| format!("Failed to stat {} after hashing", path.display()))?;
Ok(InputFingerprint::Sha256 {
sha256,
legacy_timestamp_ms: Some(timestamp_ms(metadata.modified()?)?),
})
}
fn should_walk_entry(path: &Path, skip_dirs: &BTreeSet<&str>) -> bool {
path.file_name()
.and_then(OsStr::to_str)
.is_none_or(|name| !skip_dirs.contains(name))
}
fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
fn gather_wasm_inputs(project_root: &Path) -> Result<StepInputs> {
let markdown_parser_rust_dir = PathBuf::from("../packages/markdown_parser/rust");
let mut inputs = collect_file_stats(
let mut inputs = collect_file_digests(
project_root,
&[
PathBuf::from("../tools/ci/Cargo.toml"),
@@ -361,12 +439,12 @@ fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
markdown_parser_rust_dir.join("Cargo.toml"),
],
)?;
inputs.extend(collect_directory_stats(
inputs.extend(collect_directory_digests(
project_root,
Path::new("rust/libfluxcore"),
|path| !path.contains("/target/"),
)?);
inputs.extend(collect_directory_stats(
inputs.extend(collect_directory_digests(
project_root,
&markdown_parser_rust_dir,
|path| !path.contains("/target/"),
@@ -374,15 +452,15 @@ fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
Ok(inputs)
}
fn gather_color_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_file_stats(
fn gather_color_inputs(project_root: &Path) -> Result<StepInputs> {
collect_file_digests(
project_root,
&[PathBuf::from("scripts/GenerateColorSystem.ts")],
)
}
fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_file_stats(
fn gather_message_layout_inputs(project_root: &Path) -> Result<StepInputs> {
collect_file_digests(
project_root,
&[
PathBuf::from("scripts/GenerateMessageLayoutCss.ts"),
@@ -391,8 +469,8 @@ fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String,
)
}
fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_file_stats(
fn gather_mask_inputs(project_root: &Path) -> Result<StepInputs> {
collect_file_digests(
project_root,
&[
PathBuf::from("scripts/GenerateAvatarMasks.ts"),
@@ -401,14 +479,14 @@ fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
)
}
fn gather_css_module_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_directory_stats(project_root, Path::new("src"), |path| {
fn gather_css_module_inputs(project_root: &Path) -> Result<StepInputs> {
collect_directory_digests(project_root, Path::new("src"), |path| {
path.ends_with(".module.css")
})
}
fn gather_lingui_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_directory_stats(
fn gather_lingui_inputs(project_root: &Path) -> Result<StepInputs> {
collect_directory_digests(
project_root,
Path::new("src/features/i18n/locales"),
|path| path.ends_with(".po"),
@@ -493,12 +571,6 @@ fn timestamp_ms(timestamp: SystemTime) -> Result<f64> {
* 1000.0)
}
fn env_truthy(name: &str) -> bool {
std::env::var(name)
.ok()
.is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true"))
}
fn display_command(command: &str, args: &[&str]) -> String {
std::iter::once(command.to_string())
.chain(args.iter().map(|arg| quote_arg(arg)))
tools/dev/cassandra_target_schema.json
-86
View File
@@ -9194,92 +9194,6 @@
],
"primary_key": "((guild_id), webhook_id)",
"options": ""
},
{
"name": "gateway_hotpatch_events_by_build",
"columns": [
{
"name": "build_sha",
"type": "text"
},
{
"name": "event_id",
"type": "timeuuid"
},
{
"name": "schema_version",
"type": "int"
},
{
"name": "kind",
"type": "text"
},
{
"name": "created_at",
"type": "timestamp"
},
{
"name": "created_by",
"type": "text"
},
{
"name": "signer_key_id",
"type": "text"
},
{
"name": "bundle_sha256",
"type": "blob"
},
{
"name": "signature",
"type": "blob"
},
{
"name": "bundle",
"type": "blob"
}
],
"primary_key": "((build_sha), event_id)",
"options": "CLUSTERING ORDER BY (event_id ASC)"
},
{
"name": "gateway_hotpatch_applied_by_node",
"columns": [
{
"name": "build_sha",
"type": "text"
},
{
"name": "node_name",
"type": "text"
},
{
"name": "event_id",
"type": "timeuuid"
},
{
"name": "applied_at",
"type": "timestamp"
},
{
"name": "module_count",
"type": "int"
},
{
"name": "bundle_sha256",
"type": "blob"
},
{
"name": "status",
"type": "text"
},
{
"name": "error",
"type": "text"
}
],
"primary_key": "((build_sha, node_name), event_id)",
"options": "CLUSTERING ORDER BY (event_id ASC)"
}
],
"indexes": [
tools/dev/src/bootstrap.rs
+7 -18
View File
@@ -1,18 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::desktop::install_desktop;
use crate::gateway::setup_gateway_config;
use crate::object_store::{bootstrap_schema_and_object_store, wait_s3_api};
use crate::object_store::bootstrap_schema;
use crate::paths::{ensure_state_dirs, ensure_writable_dev_paths};
use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http, wait_tcp};
use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http_success, wait_tcp};
use anyhow::Result;
pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result<()> {
pub async fn bootstrap(skip_install: bool) -> Result<()> {
ensure_state_dirs()?;
ensure_writable_dev_paths()?;
if !skip_install {
crate::proc::run(&["corepack", "enable"])?;
crate::proc::run(&["corepack", "prepare", "pnpm@10.29.3", "--activate"])?;
run_command(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
@@ -23,13 +20,10 @@ pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result
..RunOptions::default()
},
)?;
if !skip_desktop_install {
install_desktop()?;
}
}
setup_gateway_config()?;
wait_core_infra().await?;
bootstrap_schema_and_object_store().await?;
bootstrap_schema().await?;
println!("Fluxer dev bootstrap complete.");
Ok(())
}
@@ -43,16 +37,11 @@ pub async fn post_start() -> Result<()> {
}
pub async fn wait_core_infra() -> Result<()> {
wait_tcp("Postgres", "postgres", 5432, 120).await?;
wait_http_success("Meilisearch", "http://meilisearch:7700/health", 120).await?;
wait_tcp("Valkey", "valkey", 6379, 120).await?;
wait_tcp("NATS", "nats", 4222, 120).await?;
wait_tcp("LiveKit", "livekit", 7880, 120).await?;
crate::media_proxy::ensure_dev_object_store(true, 120).await?;
wait_tcp("SeaweedFS S3", "127.0.0.1", 8333, 120).await?;
wait_http(
"SeaweedFS master",
"http://127.0.0.1:9333/cluster/status",
120,
)
.await?;
wait_s3_api(120).await
Ok(())
}
tools/dev/src/dev.rs
+266 -117
View File
@@ -1,15 +1,16 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::gateway::setup_gateway_config;
use crate::gateway::{build_gateway_cluster_nodes, setup_gateway_config};
use crate::manifest::{
ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_PORT, LOOPBACK_HOST,
MEDIA_PROXY_PORT, rust_services,
ADMIN_PORT, ANY_HOST, API_PORT, APP_PORT, APP_PROXY_PORT, DEV_PROXY_GATEWAY_PORTS_ENV,
DEV_PROXY_PORT, GATEWAY_PORT, LOOPBACK_HOST, MEDIA_PROXY_PORT, rust_services,
};
use crate::object_store::s3_endpoint;
use crate::paths::{DESKTOP_DIR, ROOT};
use crate::proc::{
PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command,
merged_env, restart_budget_exceeded, run_command, wait_http,
AwaitOutcome, PNPM_INSTALL_ENV, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal,
await_or_shutdown, configure_process_group, format_command, merged_env, prefix_output,
restart_budget_exceeded, run_command_interruptible, wait_http,
};
use anyhow::{Context, Result, bail};
use std::collections::{BTreeMap, VecDeque};
@@ -30,10 +31,10 @@ const DEFAULT_TASKS: &[&str] = &[
"proxy",
"services",
"media",
"gateway",
"marketing",
"admin",
"api",
"gateway-single",
"worker",
"app",
"app-proxy",
@@ -63,7 +64,7 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
"The private marketing project is unavailable. Authorized maintainers can initialize it with:\n {MARKETING_INITIALIZE_COMMAND}"
);
}
let tasks = task_table_for_availability(marketing_availability)?;
let mut tasks = task_table_for_availability(marketing_availability)?;
let selected = if task_names.is_empty() {
DEFAULT_TASKS
.iter()
@@ -88,11 +89,30 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
unknown.join(", ")
);
}
ensure_js_dependencies_if_needed(&selected)?;
ensure_object_store_if_needed(&selected).await?;
wait_for_search_backend_if_needed(&selected).await?;
setup_gateway_config()?;
let gateway_readiness_port = configure_gateway_proxy_ports(&mut tasks, &selected)?;
let mut shutdown = ShutdownSignal::new()?;
match ensure_js_dependencies_if_needed(&selected, &mut shutdown).await? {
AwaitOutcome::Completed(()) => {}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev startup...");
return Ok(0);
}
}
match await_or_shutdown(&mut shutdown, ensure_object_store_if_needed(&selected)).await {
AwaitOutcome::Completed(result) => result?,
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev startup...");
return Ok(0);
}
}
match await_or_shutdown(&mut shutdown, wait_for_search_backend_if_needed(&selected)).await {
AwaitOutcome::Completed(result) => result?,
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev startup...");
return Ok(0);
}
}
setup_gateway_config()?;
let mut processes = Vec::new();
let mut task_names = Vec::new();
let mut task_specs = Vec::new();
@@ -102,29 +122,82 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
let mut next_object_store_check = Instant::now() + object_store_monitor_interval;
let selected_for_readiness = selected.clone();
for name in selected {
if name == CLOUDFLARE_TUNNEL_TASK
&& let Err(error) =
wait_for_cloudflare_tunnel_routes(&mut processes, &selected_for_readiness).await
{
crate::gateway::stop_processes(&mut processes);
return Err(error);
if name == CLOUDFLARE_TUNNEL_TASK {
match await_or_shutdown(
&mut shutdown,
wait_for_cloudflare_tunnel_routes(&mut processes, &selected_for_readiness),
)
.await
{
AwaitOutcome::Completed(Ok(())) => {}
AwaitOutcome::Completed(Err(error)) => {
crate::gateway::stop_processes(&mut processes);
return Err(error);
}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev tasks...");
crate::gateway::stop_processes(&mut processes);
return Ok(0);
}
}
}
let process = start_task(tasks.get(name.as_str()).expect("validated task"))?;
let process = match start_task(tasks.get(name.as_str()).expect("validated task")) {
Ok(process) => process,
Err(error) => {
crate::gateway::stop_processes(&mut processes);
return Err(error);
}
};
processes.push(process);
task_names.push(name.clone());
task_specs.push(tasks.get(name.as_str()).expect("validated task").clone());
task_restarts.push(VecDeque::new());
if name == "services"
&& let Err(error) = wait_for_rust_services(&mut processes).await
{
crate::gateway::stop_processes(&mut processes);
return Err(error);
if name == "services" {
match await_or_shutdown(&mut shutdown, wait_for_rust_services(&mut processes)).await {
AwaitOutcome::Completed(Ok(())) => {}
AwaitOutcome::Completed(Err(error)) => {
crate::gateway::stop_processes(&mut processes);
return Err(error);
}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev tasks...");
crate::gateway::stop_processes(&mut processes);
return Ok(0);
}
}
}
if name == "api"
&& let Err(error) = wait_for_api(&mut processes).await
{
crate::gateway::stop_processes(&mut processes);
return Err(error);
if name == "gateway" || name == "gateway-single" {
match await_or_shutdown(
&mut shutdown,
wait_for_gateway(&mut processes, gateway_readiness_port),
)
.await
{
AwaitOutcome::Completed(Ok(())) => {}
AwaitOutcome::Completed(Err(error)) => {
crate::gateway::stop_processes(&mut processes);
return Err(error);
}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev tasks...");
crate::gateway::stop_processes(&mut processes);
return Ok(0);
}
}
}
if name == "api" {
match await_or_shutdown(&mut shutdown, wait_for_api(&mut processes)).await {
AwaitOutcome::Completed(Ok(())) => {}
AwaitOutcome::Completed(Err(error)) => {
crate::gateway::stop_processes(&mut processes);
return Err(error);
}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev tasks...");
crate::gateway::stop_processes(&mut processes);
return Ok(0);
}
}
}
}
loop {
@@ -135,9 +208,17 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
return Err(error);
}
if monitor_object_store && Instant::now() >= next_object_store_check {
if let Err(error) = monitor_object_store_dependency().await {
crate::gateway::stop_processes(&mut processes);
return Err(error);
match await_or_shutdown(&mut shutdown, monitor_object_store_dependency()).await {
AwaitOutcome::Completed(Ok(())) => {}
AwaitOutcome::Completed(Err(error)) => {
crate::gateway::stop_processes(&mut processes);
return Err(error);
}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping dev tasks...");
crate::gateway::stop_processes(&mut processes);
return Ok(0);
}
}
next_object_store_check = Instant::now() + object_store_monitor_interval;
}
@@ -152,6 +233,38 @@ pub async fn run_dev(task_names: &[String], cloudflare_tunnel: bool) -> Result<i
}
}
fn configure_gateway_proxy_ports(
tasks: &mut BTreeMap<&'static str, DevTask>,
selected: &[String],
) -> Result<u16> {
let cluster_selected = selected.iter().any(|name| name == "gateway");
let single_selected = selected.iter().any(|name| name == "gateway-single");
if cluster_selected && single_selected {
bail!("Select either `gateway` or `gateway-single`, not both");
}
let ports = if cluster_selected {
build_gateway_cluster_nodes()?
.into_iter()
.filter(|node| node.role == "websocket")
.map(|node| node.http_port)
.collect::<Vec<_>>()
} else {
vec![GATEWAY_PORT]
};
let value = ports
.iter()
.map(u16::to_string)
.collect::<Vec<_>>()
.join(",");
let proxy = tasks
.get_mut("proxy")
.context("dev task table is missing the proxy task")?;
proxy
.env
.push((DEV_PROXY_GATEWAY_PORTS_ENV.to_owned(), Some(value)));
Ok(*ports.first().expect("gateway proxy ports are non-empty"))
}
async fn ensure_object_store_if_needed(selected: &[String]) -> Result<()> {
if !selected_needs_object_store(selected) {
return Ok(());
@@ -202,6 +315,7 @@ fn restart_exited_tasks(
let Some(status) = process.try_wait()? else {
continue;
};
crate::gateway::stop_child_processes(&mut [process]);
if restart_budget_exceeded(&mut task_restarts[index], Instant::now()) {
bail!(
"Dev task {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",
@@ -237,6 +351,10 @@ fn task_table_for_availability(
marketing_availability: MarketingAvailability,
) -> Result<BTreeMap<&'static str, DevTask>> {
let self_tool = self_tool_command()?;
let api_dir = ROOT.join("fluxer_api");
let api_tsx = api_dir.join("node_modules/.bin/tsx");
let app_dir = ROOT.join("fluxer_app");
let app_dev_server = ROOT.join("tools/ci/run.sh");
let public_url = public_url();
let marketing_endpoint = std::env::var("FLUXER_MARKETING_ENDPOINT")
.unwrap_or_else(|_| format!("{public_url}/marketing"));
@@ -260,8 +378,13 @@ fn task_table_for_availability(
});
insert(DevTask {
name: "api",
args: strings(&["pnpm", "--filter", "fluxer_api", "dev"]),
cwd: ROOT.clone(),
args: vec![
api_tsx.display().to_string(),
"watch".to_owned(),
"--clear-screen=false".to_owned(),
"src/AppEntrypoint.ts".to_owned(),
],
cwd: api_dir.clone(),
env: vec![
(
"FLUXER_S3_PUBLIC_ENDPOINT".to_owned(),
@@ -286,29 +409,26 @@ fn task_table_for_availability(
});
insert(DevTask {
name: "worker",
args: strings(&[
"pnpm",
"--filter",
"fluxer_api",
"exec",
"tsx",
"watch",
"--clear-screen=false",
"src/WorkerEntrypoint.ts",
]),
cwd: ROOT.clone(),
args: vec![
api_tsx.display().to_string(),
"watch".to_owned(),
"--clear-screen=false".to_owned(),
"src/WorkerEntrypoint.ts".to_owned(),
],
cwd: api_dir,
env: Vec::new(),
});
insert(DevTask {
name: "app",
args: strings(&["pnpm", "--filter", "fluxer_app", "dev"]),
cwd: ROOT.clone(),
args: vec![
app_dev_server.display().to_string(),
"app-dev-server".to_owned(),
],
cwd: app_dir,
env: vec![
("TOKIO_WORKER_THREADS".to_owned(), Some("4".to_owned())),
("RAYON_NUM_THREADS".to_owned(), Some("4".to_owned())),
("FLUXER_APP_DEV_PORT".to_owned(), Some(APP_PORT.to_string())),
(
"FLUXER_APP_SKIP_I18N_COMPILE".to_owned(),
Some("true".to_owned()),
),
(
"FLUXER_STATIC_CDN_ENDPOINT".to_owned(),
Some(public_url.clone()),
@@ -584,27 +704,39 @@ async fn wait_for_search_backend_if_needed(selected: &[String]) -> Result<()> {
return Ok(());
}
let env = merged_env(None, true)?;
let backend = SearchBackend::from_env(&env)?;
let search_url = env
.get("FLUXER_SEARCH_URL")
.cloned()
.unwrap_or_else(|| default_search_url(&env));
wait_http(search_backend_label(&env), &search_url, 120).await
.filter(|url| !url.trim().is_empty())
.context("Missing FLUXER_SEARCH_URL for the selected search backend")?;
wait_http(backend.label(), search_url, 120).await
}
fn default_search_url(env: &BTreeMap<String, String>) -> String {
if env
.get("FLUXER_SEARCH_ENGINE")
.is_some_and(|engine| engine == "meilisearch")
{
return "http://127.0.0.1:7700".to_owned();
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SearchBackend {
Elasticsearch,
Meilisearch,
}
impl SearchBackend {
fn from_env(env: &BTreeMap<String, String>) -> Result<Self> {
match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) {
Some("elasticsearch") => Ok(Self::Elasticsearch),
Some("meilisearch") => Ok(Self::Meilisearch),
Some(engine) => bail!(
"Unsupported FLUXER_SEARCH_ENGINE value {engine:?}; expected `meilisearch` or `elasticsearch`"
),
None => {
bail!("Missing FLUXER_SEARCH_ENGINE; expected `meilisearch` or `elasticsearch`")
}
}
}
"http://127.0.0.1:9200".to_owned()
}
fn search_backend_label(env: &BTreeMap<String, String>) -> &'static str {
match env.get("FLUXER_SEARCH_ENGINE").map(String::as_str) {
Some("meilisearch") => "Meilisearch",
_ => "Elasticsearch",
fn label(self) -> &'static str {
match self {
Self::Elasticsearch => "Elasticsearch",
Self::Meilisearch => "Meilisearch",
}
}
}
@@ -627,9 +759,12 @@ fn public_url() -> String {
.unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}"))
}
fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {
async fn ensure_js_dependencies_if_needed(
selected: &[String],
shutdown: &mut ShutdownSignal,
) -> Result<AwaitOutcome<()>> {
if selected_needs_js_dependency_preflight(selected) {
run_command(
match run_command_interruptible(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
env: PNPM_INSTALL_ENV
@@ -638,11 +773,17 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {
.collect(),
..RunOptions::default()
},
)?;
shutdown,
)
.await?
{
AwaitOutcome::Completed(_) => {}
AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)),
}
}
if selected.iter().any(|name| name == "marketing") {
let marketing_dir = ROOT.join("fluxer_marketing");
run_command(
match run_command_interruptible(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
cwd: &marketing_dir,
@@ -652,9 +793,15 @@ fn ensure_js_dependencies_if_needed(selected: &[String]) -> Result<()> {
.collect(),
..RunOptions::default()
},
)?;
shutdown,
)
.await?
{
AwaitOutcome::Completed(_) => {}
AwaitOutcome::Shutdown(signal) => return Ok(AwaitOutcome::Shutdown(signal)),
}
}
Ok(())
Ok(AwaitOutcome::Completed(()))
}
fn selected_needs_js_dependency_preflight(selected: &[String]) -> bool {
@@ -689,14 +836,7 @@ fn start_task(task: &DevTask) -> Result<Child> {
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
unsafe {
use std::os::unix::process::CommandExt;
command.pre_exec(|| {
libc::setsid();
Ok(())
});
}
configure_process_group(&mut command);
let mut child = command.spawn()?;
if let Some(stdout) = child.stdout.take() {
let label = task.name.to_owned();
@@ -718,13 +858,16 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> {
.len()
.checked_sub(1)
.expect("services process was just pushed");
let deadline = Instant::now() + Duration::from_secs(timeout);
for spec in rust_services() {
for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {
check_startup_processes(processes, services_index, spec.name, mode)?;
wait_http(
&format!("Rust service {}:{mode}", spec.name),
let name = format!("Rust service {}:{mode}", spec.name);
wait_http_with_startup_checks_until(
processes,
services_index,
&name,
&format!("http://{LOOPBACK_HOST}:{port}/_health"),
timeout,
deadline,
)
.await?;
}
@@ -732,6 +875,25 @@ async fn wait_for_rust_services(processes: &mut [Child]) -> Result<()> {
Ok(())
}
async fn wait_for_gateway(processes: &mut [Child], port: u16) -> Result<()> {
let timeout = std::env::var("FLUXER_DEV_GATEWAY_READY_TIMEOUT")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(180);
let gateway_index = processes
.len()
.checked_sub(1)
.expect("gateway process was just pushed");
wait_http_with_startup_checks(
processes,
gateway_index,
"Gateway",
&format!("http://{LOOPBACK_HOST}:{port}/_health/ready"),
timeout,
)
.await
}
async fn wait_for_api(processes: &mut [Child]) -> Result<()> {
let timeout = std::env::var("FLUXER_DEV_API_READY_TIMEOUT")
.ok()
@@ -827,7 +989,7 @@ async fn wait_http_for_dev_tasks(
while Instant::now() < deadline {
check_dev_startup_processes(processes, name)?;
match client.get(url).send().await {
Ok(response) if response.status().as_u16() < 500 => {
Ok(response) if response.status().is_success() => {
println!("{name} is reachable at {url}");
return Ok(());
}
@@ -849,16 +1011,32 @@ async fn wait_http_with_startup_checks(
name: &str,
url: &str,
timeout_secs: u64,
) -> Result<()> {
wait_http_with_startup_checks_until(
processes,
watched_index,
name,
url,
Instant::now() + Duration::from_secs(timeout_secs),
)
.await
}
async fn wait_http_with_startup_checks_until(
processes: &mut [Child],
watched_index: usize,
name: &str,
url: &str,
deadline: Instant,
) -> Result<()> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_error = None;
while Instant::now() < deadline {
check_startup_task_processes(processes, watched_index, name)?;
match client.get(url).send().await {
Ok(response) if response.status().as_u16() < 500 => {
Ok(response) if response.status().is_success() => {
println!("{name} is reachable at {url}");
return Ok(());
}
@@ -901,35 +1079,6 @@ fn check_startup_task_processes(
Ok(())
}
fn check_startup_processes(
processes: &mut [Child],
services_index: usize,
service_name: &str,
mode: &str,
) -> Result<()> {
for (index, process) in processes.iter_mut().enumerate() {
if let Some(status) = process.try_wait()? {
let code = status.code().unwrap_or(1);
if index == services_index {
bail!(
"Rust service supervisor exited with status {code} before {service_name}:{mode} became ready"
);
}
bail!(
"Dev task exited with status {code} before Rust service {service_name}:{mode} became ready"
);
}
}
Ok(())
}
fn prefix_output(label: &str, reader: impl std::io::Read) {
use std::io::{BufRead, BufReader};
for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
println!("[{label}] {line}");
}
}
#[allow(dead_code)]
fn _cwd_is_root(path: &Path) -> bool {
path == ROOT.as_path()
@@ -940,17 +1089,17 @@ mod tests {
use super::*;
#[test]
fn default_tasks_keep_legacy_order() {
fn default_tasks_use_lightweight_gateway() {
assert_eq!(
DEFAULT_TASKS,
&[
"proxy",
"services",
"media",
"gateway",
"marketing",
"admin",
"api",
"gateway-single",
"worker",
"app",
"app-proxy"
tools/dev/src/gateway.rs
+392 -319
View File
@@ -1,18 +1,20 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::gateway_reload::{
ArtifactState, changed_artifacts, hot_reload_enabled, hot_reload_modules, snapshot_artifacts,
spawn_source_watcher,
};
use crate::paths::{DEV_GATEWAY_DIR, GATEWAY_CONFIG_DIR, ROOT};
use crate::proc::{
RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,
restart_budget_exceeded, run_command,
AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown,
configure_process_group, force_kill_process_group, format_command, merged_env,
pending_shutdown, prefix_output, process_group_running, restart_budget_exceeded,
run_command_interruptible, stop_process_group, terminate_process_group,
};
use anyhow::{Context, Result, bail};
use sha2::{Digest, Sha256};
#[cfg(target_os = "linux")]
use std::collections::BTreeSet;
use std::collections::VecDeque;
use std::env;
use std::fs;
use std::io::Read;
use std::net::{SocketAddr, TcpListener};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
@@ -47,9 +49,14 @@ const GATEWAY_COMPILE_COMMAND: &[&str] = &[
"--step",
"compile",
];
const GATEWAY_DEPENDENCY_INPUTS: &[&str] = &["rebar.config", "rebar.config.script", "rebar.lock"];
const GATEWAY_DEPENDENCY_INPUT_MAX_BYTES: u64 = 4 * 1024 * 1024;
const GATEWAY_DEPENDENCY_STAMP_FILE: &str = "rebar-dependencies.sha256";
const GATEWAY_DEPENDENCY_STAMP_MAX_BYTES: u64 = 128;
const NODE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
const NODE_RESTART_PORT_WAIT: Duration = Duration::from_secs(75);
const STOP_GRACE_PERIOD: Duration = Duration::from_secs(10);
const NESTED_STOP_GRACE_PERIOD: Duration = Duration::from_secs(5);
const ROOT_STOP_GRACE_PERIOD: Duration = Duration::from_secs(15);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GatewayNode {
@@ -128,33 +135,42 @@ fn remove_stale_gateway_config() -> Result<()> {
Ok(())
}
pub fn run_gateway() -> Result<()> {
setup_gateway_config()?;
compile_gateway()?;
let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?;
let env = merged_env(None, true)?;
println!("$ {}", format_command(&command));
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
let err = Command::new(&command[0])
.args(&command[1..])
.current_dir(gateway_dir())
.env_clear()
.envs(env)
.exec();
bail!("failed to exec gateway: {err}");
pub async fn run_gateway() -> Result<i32> {
let mut shutdown = ShutdownSignal::new()?;
let node_names = std::collections::HashSet::from([env::var("FLUXER_ERLANG_NODE_NAME")
.unwrap_or_else(|_| "fluxer_gateway@127.0.0.1".to_owned())]);
cleanup_orphaned_gateway_processes(&node_names).await?;
if let Some(signal) = pending_shutdown(&mut shutdown).await {
println!("Received {signal}; stopping gateway startup...");
return Ok(0);
}
#[cfg(not(unix))]
{
let status = Command::new(&command[0])
.args(&command[1..])
.current_dir(gateway_dir())
.env_clear()
.envs(env)
.status()?;
std::process::exit(status.code().unwrap_or(1));
setup_gateway_config()?;
match compile_gateway_interruptible(&mut shutdown).await? {
AwaitOutcome::Completed(_) => {}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping gateway startup...");
return Ok(0);
}
}
let command = build_gateway_command(DEV_GATEWAY_DIR.as_path())?;
let command_refs = command.iter().map(String::as_str).collect::<Vec<_>>();
let working_dir = gateway_dir();
let outcome = run_command_interruptible(
&command_refs,
RunOptions {
cwd: &working_dir,
..RunOptions::default()
},
&mut shutdown,
)
.await?;
stop_idle_epmd();
match outcome {
AwaitOutcome::Completed(output) => Ok(output.status.code().unwrap_or(1)),
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping gateway...");
Ok(0)
}
}
}
@@ -166,11 +182,31 @@ struct SupervisedNode {
pub async fn run_gateway_cluster() -> Result<i32> {
let nodes = build_gateway_cluster_nodes()?;
cleanup_orphaned_gateway_nodes(&nodes).await?;
wait_for_cluster_ports_available(&nodes).await?;
setup_gateway_cluster_config(&nodes)?;
compile_gateway()?;
let mut shutdown = ShutdownSignal::new()?;
let node_names = nodes
.iter()
.map(GatewayNode::erlang_name)
.collect::<std::collections::HashSet<_>>();
cleanup_orphaned_gateway_processes(&node_names).await?;
if let Some(signal) = pending_shutdown(&mut shutdown).await {
println!("Received {signal}; stopping gateway startup...");
return Ok(0);
}
match await_or_shutdown(&mut shutdown, wait_for_cluster_ports_available(&nodes)).await {
AwaitOutcome::Completed(result) => result?,
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping gateway startup...");
return Ok(0);
}
}
setup_gateway_cluster_config(&nodes)?;
match compile_gateway_interruptible(&mut shutdown).await? {
AwaitOutcome::Completed(_) => {}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping gateway startup...");
return Ok(0);
}
}
let static_peers = nodes
.iter()
.map(GatewayNode::erlang_name)
@@ -179,57 +215,35 @@ pub async fn run_gateway_cluster() -> Result<i32> {
let mut supervised = Vec::new();
print_gateway_cluster_topology(&nodes);
for node in &nodes {
let child = match start_node(node, &static_peers) {
Ok(child) => child,
Err(error) => {
stop_supervised(&mut supervised);
return Err(error);
}
};
supervised.push(SupervisedNode {
node: node.clone(),
child: start_node(node, &static_peers)?,
child,
restarts: VecDeque::new(),
});
}
let watcher = hot_reload_enabled().then(spawn_source_watcher);
if watcher.is_some() {
println!(
"Gateway hot reload enabled; watching fluxer_gateway sources (set FLUXER_DEV_GATEWAY_HOT_RELOAD=false to disable)"
);
}
let mut compile: Option<(tokio::task::JoinHandle<bool>, ArtifactState)> = None;
let mut compile_queued = false;
loop {
if let Err(error) = restart_exited_nodes(&mut supervised, &static_peers).await {
stop_supervised(&mut supervised);
return Err(error);
}
if let Some(receiver) = &watcher {
while receiver.try_recv().is_ok() {
compile_queued = true;
}
}
if compile_queued && compile.is_none() {
compile_queued = false;
println!("Gateway sources changed; recompiling for hot reload...");
let before = snapshot_artifacts();
compile = Some((
tokio::task::spawn_blocking(compile_gateway_for_reload),
before,
));
}
if compile
.as_ref()
.is_some_and(|(handle, _)| handle.is_finished())
match await_or_shutdown(
&mut shutdown,
restart_exited_nodes(&mut supervised, &static_peers),
)
.await
{
let (handle, before) = compile.take().expect("compile task present");
match handle.await {
Ok(true) => {
if let Err(error) =
apply_hot_reload(&mut supervised, &before, &static_peers).await
{
stop_supervised(&mut supervised);
return Err(error);
}
}
Ok(false) => println!(
"Gateway compile failed; hot reload skipped (fix the errors and save again)"
),
Err(error) => println!("Gateway compile task failed: {error}"),
AwaitOutcome::Completed(Ok(())) => {}
AwaitOutcome::Completed(Err(error)) => {
stop_supervised(&mut supervised);
return Err(error);
}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping gateway cluster...");
stop_supervised(&mut supervised);
return Ok(0);
}
}
tokio::select! {
@@ -243,24 +257,116 @@ pub async fn run_gateway_cluster() -> Result<i32> {
}
}
fn compile_gateway() -> Result<()> {
run_command(GATEWAY_COMPILE_COMMAND, RunOptions::default()).map(drop)
async fn compile_gateway_interruptible(
shutdown: &mut ShutdownSignal,
) -> Result<AwaitOutcome<std::process::Output>> {
let dependency_stamp = prepare_gateway_compile()?;
let outcome =
run_command_interruptible(GATEWAY_COMPILE_COMMAND, RunOptions::default(), shutdown).await?;
if matches!(&outcome, AwaitOutcome::Completed(_)) {
write_gateway_dependency_stamp(&dependency_stamp)?;
}
Ok(outcome)
}
fn compile_gateway_for_reload() -> bool {
run_command(
GATEWAY_COMPILE_COMMAND,
RunOptions {
check: false,
..RunOptions::default()
},
)
.map(|output| output.status.success())
.unwrap_or(false)
fn prepare_gateway_compile() -> Result<String> {
let dependency_stamp = gateway_dependency_stamp()?;
if gateway_dependency_stamp_matches(&dependency_stamp)? {
return Ok(dependency_stamp);
}
let build_dir = gateway_dir().join("_build/default");
println!(
"Gateway dependency inputs changed; removing stale {}",
build_dir.display()
);
match fs::remove_dir_all(&build_dir) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => {
return Err(error).with_context(|| format!("failed to remove {}", build_dir.display()));
}
}
Ok(dependency_stamp)
}
fn cluster_cookie() -> String {
env::var("FLUXER_ERLANG_COOKIE").unwrap_or_else(|_| GATEWAY_CLUSTER_COOKIE.to_owned())
fn gateway_dependency_stamp() -> Result<String> {
let mut hasher = Sha256::new();
let mut total_bytes = 0_u64;
for input in GATEWAY_DEPENDENCY_INPUTS {
let path = gateway_dir().join(input);
let metadata = fs::metadata(&path)
.with_context(|| format!("failed to read metadata for {}", path.display()))?;
if !metadata.is_file() {
bail!("gateway dependency input is not a file: {}", path.display());
}
total_bytes = total_bytes
.checked_add(metadata.len())
.context("gateway dependency input byte count overflowed")?;
if total_bytes > GATEWAY_DEPENDENCY_INPUT_MAX_BYTES {
bail!(
"gateway dependency inputs exceed {} bytes",
GATEWAY_DEPENDENCY_INPUT_MAX_BYTES
);
}
let mut bytes = Vec::with_capacity(metadata.len() as usize);
fs::File::open(&path)
.with_context(|| format!("failed to open {}", path.display()))?
.take(metadata.len() + 1)
.read_to_end(&mut bytes)
.with_context(|| format!("failed to read {}", path.display()))?;
if bytes.len() as u64 != metadata.len() {
bail!(
"gateway dependency input changed while reading: {}",
path.display()
);
}
hasher.update((input.len() as u64).to_le_bytes());
hasher.update(input.as_bytes());
hasher.update(metadata.len().to_le_bytes());
hasher.update(&bytes);
}
Ok(bytes_to_lower_hex(&hasher.finalize()))
}
fn bytes_to_lower_hex(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(bytes.len() * 2);
for byte in bytes {
output.push(HEX[(byte >> 4) as usize] as char);
output.push(HEX[(byte & 0x0f) as usize] as char);
}
output
}
fn gateway_dependency_stamp_matches(expected: &str) -> Result<bool> {
let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE);
let file = match fs::File::open(&path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(error) => {
return Err(error).with_context(|| format!("failed to open {}", path.display()));
}
};
let mut bytes = Vec::new();
file.take(GATEWAY_DEPENDENCY_STAMP_MAX_BYTES + 1)
.read_to_end(&mut bytes)
.with_context(|| format!("failed to read {}", path.display()))?;
if bytes.len() as u64 > GATEWAY_DEPENDENCY_STAMP_MAX_BYTES {
bail!(
"gateway dependency stamp exceeds {} bytes: {}",
GATEWAY_DEPENDENCY_STAMP_MAX_BYTES,
path.display()
);
}
Ok(bytes == format!("{expected}\n").as_bytes())
}
fn write_gateway_dependency_stamp(dependency_stamp: &str) -> Result<()> {
fs::create_dir_all(DEV_GATEWAY_DIR.as_path())
.with_context(|| format!("failed to create {}", DEV_GATEWAY_DIR.display()))?;
let path = DEV_GATEWAY_DIR.join(GATEWAY_DEPENDENCY_STAMP_FILE);
fs::write(&path, format!("{dependency_stamp}\n"))
.with_context(|| format!("failed to write {}", path.display()))
}
async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: &str) -> Result<()> {
@@ -285,18 +391,7 @@ async fn restart_exited_nodes(supervised: &mut [SupervisedNode], static_peers: &
}
async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result<()> {
if entry.child.try_wait()?.is_none() {
terminate_process(&mut entry.child);
let deadline = Instant::now() + NODE_SHUTDOWN_TIMEOUT;
while entry.child.try_wait()?.is_none() {
if Instant::now() >= deadline {
let _ = entry.child.kill();
let _ = entry.child.wait();
break;
}
sleep(Duration::from_millis(100)).await;
}
}
stop_process_group(&mut entry.child, NODE_SHUTDOWN_TIMEOUT).await;
wait_for_ports_available_until(
std::slice::from_ref(&entry.node),
Instant::now() + NODE_RESTART_PORT_WAIT,
@@ -306,79 +401,13 @@ async fn restart_node(entry: &mut SupervisedNode, static_peers: &str) -> Result<
Ok(())
}
async fn apply_hot_reload(
supervised: &mut [SupervisedNode],
before: &ArtifactState,
static_peers: &str,
) -> Result<()> {
let after = snapshot_artifacts();
let diff = changed_artifacts(before, &after);
if diff.nifs_changed {
println!("Gateway native NIF artifacts changed; rolling restart of all gateway nodes...");
for entry in supervised.iter_mut() {
println!("[gateway:{}] restarting for NIF reload", entry.node.name());
restart_node(entry, static_peers).await?;
}
println!("Gateway rolling restart complete");
return Ok(());
}
if diff.modules.is_empty() {
println!("Gateway compile finished; no module changes to reload");
return Ok(());
}
println!(
"Hot reloading {} gateway module(s) across {} node(s): {}",
diff.modules.len(),
supervised.len(),
diff.modules.join(", ")
);
let nodes = supervised
.iter()
.map(|entry| entry.node.clone())
.collect::<Vec<_>>();
let modules = diff.modules.clone();
let cookie = cluster_cookie();
let outcome =
tokio::task::spawn_blocking(move || hot_reload_modules(&nodes, &modules, &cookie)).await;
match outcome {
Ok(Ok(outcome)) if outcome.failed_nodes.is_empty() => {
println!(
"Gateway hot reload complete ({} module(s) live)",
diff.modules.len()
);
}
Ok(Ok(outcome)) => {
for entry in supervised.iter_mut() {
if !outcome.failed_nodes.contains(&entry.node.erlang_name()) {
continue;
}
if entry.child.try_wait()?.is_some() {
continue;
}
println!(
"[gateway:{}] hot reload failed; restarting node to pick up new code",
entry.node.name()
);
restart_node(entry, static_peers).await?;
}
}
Ok(Err(error)) => {
println!("Gateway hot reload failed: {error}; rolling restart of all gateway nodes");
for entry in supervised.iter_mut() {
restart_node(entry, static_peers).await?;
}
}
Err(error) => println!("Gateway hot reload task failed: {error}"),
}
Ok(())
}
fn stop_supervised(supervised: &mut [SupervisedNode]) {
let mut children = supervised
.iter_mut()
.map(|entry| &mut entry.child)
.collect::<Vec<_>>();
stop_child_processes(&mut children);
stop_idle_epmd();
}
pub fn build_gateway_cluster_nodes() -> Result<Vec<GatewayNode>> {
@@ -492,18 +521,20 @@ fn can_bind_tcp_port(port: u16) -> bool {
}
#[cfg(target_os = "linux")]
async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> {
let leaders = orphaned_gateway_leaders(nodes)?;
async fn cleanup_orphaned_gateway_processes(
node_names: &std::collections::HashSet<String>,
) -> Result<()> {
let leaders = orphaned_gateway_leaders(node_names)?;
if leaders.is_empty() {
return Ok(());
}
let pids = leaders.iter().map(|leader| leader.pid).collect::<Vec<_>>();
println!(
"Stopping orphaned gateway cluster node process group(s): {}",
"Stopping orphaned gateway process group(s): {}",
format_pids(&pids)
);
let term_failed = signal_process_groups(&pids, libc::SIGTERM);
let term_failed = signal_process_groups(&leaders, libc::SIGTERM);
if !term_failed.is_empty() {
println!(
"SIGTERM delivery failed for orphaned gateway pid(s): {}",
@@ -518,7 +549,12 @@ async fn cleanup_orphaned_gateway_nodes(nodes: &[GatewayNode]) -> Result<()> {
return Ok(());
}
if Instant::now() >= deadline {
let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);
let remaining_leaders = leaders
.iter()
.filter(|leader| remaining.contains(&leader.pid))
.cloned()
.collect::<Vec<_>>();
let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL);
sleep(Duration::from_millis(200)).await;
let survivors = surviving_gateway_group_pids(&leaders)?;
if survivors.is_empty() {
@@ -556,24 +592,95 @@ fn format_pids(pids: &[i32]) -> String {
}
#[cfg(not(target_os = "linux"))]
async fn cleanup_orphaned_gateway_nodes(_nodes: &[GatewayNode]) -> Result<()> {
async fn cleanup_orphaned_gateway_processes(
_node_names: &std::collections::HashSet<String>,
) -> Result<()> {
Ok(())
}
#[cfg(target_os = "linux")]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq)]
struct GatewayLeader {
pid: i32,
members: Vec<GatewayProcessIdentity>,
}
#[cfg(target_os = "linux")]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct GatewayProcessIdentity {
pid: i32,
starttime: u64,
}
#[cfg(target_os = "linux")]
fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result<Vec<GatewayLeader>> {
let node_names = nodes
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct GatewayProcess {
pid: i32,
ppid: i32,
pgid: i32,
state: char,
starttime: u64,
}
#[cfg(target_os = "linux")]
fn orphaned_gateway_leaders(
node_names: &std::collections::HashSet<String>,
) -> Result<Vec<GatewayLeader>> {
let current_exe = std::env::current_exe().context("failed to resolve fluxer-dev executable")?;
let processes = gateway_process_snapshot()?;
let mut owned_pids = BTreeSet::new();
for process in &processes {
if process.ppid != 1 || proc_stat_state_is_dead(process.state) {
continue;
}
let args = proc_cmdline(process.pid);
let orphaned_supervisor = cmdline_is_gateway_supervisor(&args, &current_exe);
let orphaned_node = (cmdline_has_gateway_node(&args, node_names)
|| cmdline_has_managed_gateway_config(&args))
&& fs::read_link(format!("/proc/{}/cwd", process.pid))
.ok()
.as_deref()
== Some(gateway_dir().as_path());
if orphaned_supervisor || orphaned_node {
owned_pids.insert(process.pid);
}
}
loop {
let previous_count = owned_pids.len();
for process in &processes {
if owned_pids.contains(&process.ppid) {
owned_pids.insert(process.pid);
}
}
if owned_pids.len() == previous_count {
break;
}
}
let group_pids = processes
.iter()
.map(GatewayNode::erlang_name)
.collect::<std::collections::HashSet<_>>();
let mut leaders = Vec::new();
.filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1)
.map(|process| process.pgid)
.collect::<BTreeSet<_>>();
let leaders = group_pids
.into_iter()
.map(|pid| GatewayLeader {
pid,
members: processes
.iter()
.filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid)
.map(|process| GatewayProcessIdentity {
pid: process.pid,
starttime: process.starttime,
})
.collect(),
})
.collect::<Vec<_>>();
Ok(leaders)
}
#[cfg(target_os = "linux")]
fn gateway_process_snapshot() -> Result<Vec<GatewayProcess>> {
let mut processes = Vec::new();
for entry in fs::read_dir("/proc").context("failed to read /proc")? {
let Ok(entry) = entry else {
continue;
@@ -585,18 +692,39 @@ fn orphaned_gateway_leaders(nodes: &[GatewayNode]) -> Result<Vec<GatewayLeader>>
else {
continue;
};
if !cmdline_has_gateway_node(&proc_cmdline(pid), &node_names) {
let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
continue;
}
if proc_parent_pid(pid) != Some(1) {
};
let Some((state, ppid, pgid, starttime)) = parse_proc_stat_process(&stat) else {
continue;
}
if let Some(starttime) = proc_stat_starttime(pid) {
leaders.push(GatewayLeader { pid, starttime });
}
};
processes.push(GatewayProcess {
pid,
ppid,
pgid,
state,
starttime,
});
}
leaders.sort_unstable_by_key(|leader| leader.pid);
Ok(leaders)
Ok(processes)
}
#[cfg(target_os = "linux")]
fn cmdline_is_gateway_supervisor(args: &[String], current_exe: &Path) -> bool {
args.first().map(Path::new) == Some(current_exe)
&& args.get(1).map(String::as_str) == Some("gateway")
&& match args.get(2).map(String::as_str) {
None | Some("cluster" | "single") => args.len() <= 3,
Some(_) => false,
}
}
#[cfg(target_os = "linux")]
fn cmdline_has_managed_gateway_config(args: &[String]) -> bool {
args.windows(2).any(|pair| {
matches!(pair[0].as_str(), "-config" | "-args_file")
&& Path::new(&pair[1]).starts_with(DEV_GATEWAY_DIR.as_path())
})
}
#[cfg(any(target_os = "linux", test))]
@@ -618,30 +746,6 @@ fn proc_cmdline(pid: i32) -> Vec<String> {
.collect()
}
#[cfg(target_os = "linux")]
fn proc_parent_pid(pid: i32) -> Option<i32> {
fs::read_to_string(format!("/proc/{pid}/status"))
.ok()?
.lines()
.find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())
}
#[cfg(target_os = "linux")]
fn process_exists(pid: i32) -> bool {
assert!(pid > 0);
match proc_stat_state_and_pgid(pid) {
Some((state, _pgid)) => !proc_stat_state_is_dead(state),
None => false,
}
}
#[cfg(target_os = "linux")]
fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {
assert!(pid > 0);
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
parse_proc_stat_state_and_pgid(&stat)
}
#[cfg(any(target_os = "linux", test))]
fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
let (_, after_comm) = stat.rsplit_once(')')?;
@@ -653,10 +757,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
}
#[cfg(target_os = "linux")]
fn proc_stat_starttime(pid: i32) -> Option<u64> {
assert!(pid > 0);
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
parse_proc_stat_starttime(&stat)
fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> {
let (_, after_comm) = stat.rsplit_once(')')?;
let mut fields = after_comm.split_ascii_whitespace();
let state = fields.next()?.chars().next()?;
let ppid = fields.next()?.parse().ok()?;
let pgid = fields.next()?.parse().ok()?;
Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?))
}
#[cfg(any(target_os = "linux", test))]
@@ -672,62 +779,43 @@ fn proc_stat_state_is_dead(state: char) -> bool {
#[cfg(target_os = "linux")]
fn surviving_gateway_group_pids(leaders: &[GatewayLeader]) -> Result<Vec<i32>> {
let leader_pids = leaders
Ok(leaders
.iter()
.filter(|leader| gateway_group_has_owned_member(leader))
.map(|leader| leader.pid)
.collect::<std::collections::HashSet<_>>();
let mut survivors = Vec::new();
for entry in fs::read_dir("/proc").context("failed to read /proc")? {
let Ok(entry) = entry else {
continue;
};
let Some(pid) = entry
.file_name()
.to_str()
.and_then(|name| name.parse::<i32>().ok())
else {
continue;
};
let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
continue;
};
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
continue;
};
if proc_stat_state_is_dead(state) {
continue;
}
if leader_pids.contains(&pgid) {
survivors.push(pid);
continue;
}
let leader = leaders.iter().find(|leader| leader.pid == pid);
if let Some(leader) = leader
&& parse_proc_stat_starttime(&stat) == Some(leader.starttime)
{
survivors.push(pid);
}
}
survivors.sort_unstable();
Ok(survivors)
.collect())
}
#[cfg(target_os = "linux")]
fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {
fn gateway_group_has_owned_member(leader: &GatewayLeader) -> bool {
leader.members.iter().any(|member| {
let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else {
return false;
};
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
return false;
};
!proc_stat_state_is_dead(state)
&& pgid == leader.pid
&& parse_proc_stat_starttime(&stat) == Some(member.starttime)
})
}
#[cfg(target_os = "linux")]
fn signal_process_groups(leaders: &[GatewayLeader], signal: i32) -> Vec<i32> {
assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);
let mut failed = Vec::with_capacity(pids.len());
for pid in pids {
assert!(*pid > 0);
let group_result = unsafe { libc::kill(-pid, signal) };
let mut failed = Vec::with_capacity(leaders.len());
for leader in leaders {
assert!(leader.pid > 1);
if !gateway_group_has_owned_member(leader) {
continue;
}
let group_result = unsafe { libc::kill(-leader.pid, signal) };
if group_result == 0 {
continue;
}
let direct_result = unsafe { libc::kill(*pid, signal) };
if direct_result == 0 {
continue;
}
if process_exists(*pid) {
failed.push(*pid);
if gateway_group_has_owned_member(leader) {
failed.push(leader.pid);
}
}
failed
@@ -812,14 +900,7 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result<Child> {
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
unsafe {
use std::os::unix::process::CommandExt;
child_command.pre_exec(|| {
libc::setsid();
Ok(())
});
}
configure_process_group(&mut child_command);
let mut child = child_command.spawn()?;
if let Some(stdout) = child.stdout.take() {
let label = node.name();
@@ -832,30 +913,30 @@ fn start_node(node: &GatewayNode, static_peers: &str) -> Result<Child> {
Ok(child)
}
fn prefix_output(label: &str, reader: impl std::io::Read) {
use std::io::{BufRead, BufReader};
for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
println!("[{label}] {line}");
}
}
pub fn stop_processes(processes: &mut [Child]) {
let mut children = processes.iter_mut().collect::<Vec<_>>();
stop_child_processes(&mut children);
stop_child_processes_with_grace(&mut children, ROOT_STOP_GRACE_PERIOD);
stop_idle_epmd();
}
pub fn stop_child_processes(processes: &mut [&mut Child]) {
stop_child_processes_with_grace(processes, NESTED_STOP_GRACE_PERIOD);
}
fn stop_child_processes_with_grace(processes: &mut [&mut Child], grace_period: Duration) {
for process in processes.iter_mut() {
if process.try_wait().ok().flatten().is_some() {
continue;
if let Err(error) = terminate_process_group(process) {
eprintln!(
"Failed to terminate process group {}: {error}",
process.id()
);
}
terminate_process(process);
}
let deadline = Instant::now() + STOP_GRACE_PERIOD;
let deadline = Instant::now() + grace_period;
loop {
let all_exited = processes
.iter_mut()
.all(|process| process.try_wait().ok().flatten().is_some());
let all_exited = processes.iter_mut().all(|process| {
process.try_wait().ok().flatten().is_some() && !process_group_running(process)
});
if all_exited {
return;
}
@@ -865,38 +946,30 @@ pub fn stop_child_processes(processes: &mut [&mut Child]) {
std::thread::sleep(Duration::from_millis(100));
}
for process in processes {
if process.try_wait().ok().flatten().is_some() {
continue;
if process.try_wait().ok().flatten().is_none() || process_group_running(process) {
force_kill_process_group(process);
}
force_kill_process(process);
}
}
#[cfg(unix)]
fn terminate_process(process: &mut Child) {
unsafe {
libc::kill(-(process.id() as i32), libc::SIGTERM);
fn stop_idle_epmd() {
let output = match Command::new("epmd").arg("-kill").output() {
Ok(output) => output,
Err(error) => {
eprintln!("Failed to stop Erlang port mapper: {error}");
return;
}
};
if output.status.success() {
return;
}
}
#[cfg(not(unix))]
fn terminate_process(process: &mut Child) {
let _ = process.kill();
}
#[cfg(unix)]
fn force_kill_process(process: &mut Child) {
unsafe {
libc::kill(-(process.id() as i32), libc::SIGKILL);
let mut message = output.stdout;
message.extend(output.stderr);
let message = String::from_utf8_lossy(&message);
if message.contains("Cannot connect to local epmd") {
return;
}
let _ = process.kill();
let _ = process.wait();
}
#[cfg(not(unix))]
fn force_kill_process(process: &mut Child) {
let _ = process.kill();
let _ = process.wait();
eprintln!("Failed to stop Erlang port mapper: {}", message.trim());
}
pub fn build_gateway_command(config_dir: &Path) -> Result<Vec<String>> {
tools/dev/src/gateway_reload.rs
-420
View File
@@ -1,420 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::gateway::{GatewayNode, gateway_dir};
use anyhow::{Context, Result, bail};
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::mpsc::{Receiver, channel};
use std::time::{Duration, Instant, SystemTime};
const WATCH_POLL_INTERVAL: Duration = Duration::from_millis(1000);
const WATCH_SETTLE_INTERVAL: Duration = Duration::from_millis(300);
const RELOAD_TIMEOUT: Duration = Duration::from_secs(60);
const WATCHED_SOURCE_EXTENSIONS: &[&str] = &["erl", "hrl", "src", "rs", "toml", "config"];
pub type FileState = BTreeMap<PathBuf, (SystemTime, u64)>;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ArtifactState {
pub beams: FileState,
pub nifs: FileState,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ArtifactDiff {
pub modules: Vec<String>,
pub nifs_changed: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReloadOutcome {
pub failed_nodes: Vec<String>,
}
pub fn hot_reload_enabled() -> bool {
env::var("FLUXER_DEV_GATEWAY_HOT_RELOAD")
.map(|value| !matches!(value.to_ascii_lowercase().as_str(), "0" | "false" | "no"))
.unwrap_or(true)
}
pub fn spawn_source_watcher() -> Receiver<()> {
let (sender, receiver) = channel();
std::thread::spawn(move || {
let mut previous = scan_sources();
loop {
std::thread::sleep(WATCH_POLL_INTERVAL);
let mut current = scan_sources();
if current == previous {
continue;
}
loop {
std::thread::sleep(WATCH_SETTLE_INTERVAL);
let settled = scan_sources();
if settled == current {
break;
}
current = settled;
}
previous = current;
if sender.send(()).is_err() {
return;
}
}
});
receiver
}
fn scan_sources() -> FileState {
let dir = gateway_dir();
let mut state = FileState::new();
for root in [dir.join("src"), dir.join("include"), dir.join("native")] {
scan_tree(&mut state, &root);
}
for path in [
dir.join("rebar.config"),
dir.join("rebar.config.script"),
dir.join("rebar.lock"),
] {
record_file(&mut state, &path);
}
state
}
fn scan_tree(state: &mut FileState, root: &Path) {
let Ok(entries) = fs::read_dir(root) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
let Ok(metadata) = entry.metadata() else {
continue;
};
if metadata.is_dir() {
if !is_skipped_dir(&path) {
scan_tree(state, &path);
}
} else if is_watched_source(&path)
&& let Ok(modified) = metadata.modified()
{
state.insert(path, (modified, metadata.len()));
}
}
}
fn record_file(state: &mut FileState, path: &Path) {
if let Ok(metadata) = fs::metadata(path)
&& let Ok(modified) = metadata.modified()
{
state.insert(path.to_path_buf(), (modified, metadata.len()));
}
}
fn is_skipped_dir(path: &Path) -> bool {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name == "target" || name.starts_with('.'))
}
fn is_watched_source(path: &Path) -> bool {
path.extension()
.and_then(|extension| extension.to_str())
.is_some_and(|extension| WATCHED_SOURCE_EXTENSIONS.contains(&extension))
}
pub fn snapshot_artifacts() -> ArtifactState {
let mut state = ArtifactState::default();
let lib_root = gateway_dir().join("_build/default/lib");
if let Ok(entries) = fs::read_dir(&lib_root) {
for entry in entries.flatten() {
scan_artifact_dir(&mut state.beams, &entry.path().join("ebin"), "beam");
}
}
scan_artifact_dir(&mut state.nifs, &gateway_dir().join("priv"), "so");
state
}
fn scan_artifact_dir(state: &mut FileState, dir: &Path, extension: &str) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path
.extension()
.is_some_and(|candidate| candidate == extension)
&& let Ok(metadata) = entry.metadata()
&& let Ok(modified) = metadata.modified()
{
state.insert(path, (modified, metadata.len()));
}
}
}
pub fn changed_artifacts(before: &ArtifactState, after: &ArtifactState) -> ArtifactDiff {
let modules = after
.beams
.iter()
.filter(|(path, state)| before.beams.get(*path) != Some(*state))
.filter_map(|(path, _)| Some(path.file_stem()?.to_str()?.to_owned()))
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let nifs_changed = after
.nifs
.iter()
.any(|(path, state)| before.nifs.get(path) != Some(state));
ArtifactDiff {
modules,
nifs_changed,
}
}
pub fn build_reload_eval(nodes: &[String], modules: &[String]) -> String {
let node_list = nodes
.iter()
.map(|node| format!("'{node}'"))
.collect::<Vec<_>>()
.join(",");
let module_list = modules
.iter()
.map(|module| format!("'{module}'"))
.collect::<Vec<_>>()
.join(",");
format!(
"Nodes = [{node_list}], \
Mods = [{module_list}], \
Failed = [Node || Node <- Nodes, \
case net_adm:ping(Node) of \
pong -> \
Errors = [Mod || Mod <- Mods, \
begin \
rpc:call(Node, code, purge, [Mod], 10000), \
case rpc:call(Node, code, load_file, [Mod], 10000) of \
{{module, Mod}} -> false; \
Other -> \
io:format(\"gateway_reload_error ~s ~s ~0p~n\", [Node, Mod, Other]), \
true \
end \
end], \
Errors =/= []; \
pang -> \
io:format(\"gateway_reload_node_down ~s~n\", [Node]), \
true \
end], \
[io:format(\"gateway_reload_failed_node ~s~n\", [Failed1]) || Failed1 <- Failed], \
io:format(\"gateway_reload_done~n\"), \
halt(0)."
)
}
pub fn hot_reload_modules(
nodes: &[GatewayNode],
modules: &[String],
cookie: &str,
) -> Result<ReloadOutcome> {
assert!(!modules.is_empty());
let node_names = nodes
.iter()
.map(GatewayNode::erlang_name)
.collect::<Vec<_>>();
let eval = build_reload_eval(&node_names, modules);
let reloader_name = format!("fluxer_dev_reload_{}@127.0.0.1", std::process::id());
let mut child = Command::new("erl")
.args([
"-hidden",
"-noshell",
"-name",
&reloader_name,
"-setcookie",
cookie,
"-eval",
&eval,
])
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("failed to spawn gateway reload shell")?;
let stdout = child.stdout.take().map(spawn_reader);
let stderr = child.stderr.take().map(spawn_reader);
let status = wait_with_deadline(&mut child, RELOAD_TIMEOUT)?;
let mut output = String::new();
for handle in [stdout, stderr].into_iter().flatten() {
if let Ok(text) = handle.join() {
output.push_str(&text);
}
}
for line in output.lines().filter(|line| !line.trim().is_empty()) {
println!("[gateway:reload] {line}");
}
if !status.success() {
bail!(
"gateway reload shell exited with status {}",
status.code().unwrap_or(1)
);
}
if !output.lines().any(|line| line == "gateway_reload_done") {
bail!("gateway reload shell did not report completion");
}
Ok(parse_reload_outcome(&output))
}
pub fn parse_reload_outcome(output: &str) -> ReloadOutcome {
let failed_nodes = output
.lines()
.filter_map(|line| line.strip_prefix("gateway_reload_failed_node "))
.map(|node| node.trim().to_owned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
ReloadOutcome { failed_nodes }
}
fn spawn_reader(stream: impl Read + Send + 'static) -> std::thread::JoinHandle<String> {
std::thread::spawn(move || {
let mut reader = stream;
let mut text = String::new();
let mut bytes = Vec::new();
if reader.read_to_end(&mut bytes).is_ok() {
text = String::from_utf8_lossy(&bytes).into_owned();
}
text
})
}
fn wait_with_deadline(child: &mut Child, timeout: Duration) -> Result<std::process::ExitStatus> {
let deadline = Instant::now() + timeout;
loop {
if let Some(status) = child.try_wait()? {
return Ok(status);
}
if Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
bail!(
"gateway reload shell timed out after {}s",
timeout.as_secs()
);
}
std::thread::sleep(Duration::from_millis(100));
}
}
#[cfg(test)]
mod tests {
use super::*;
fn file_state(entries: &[(&str, u64, u64)]) -> FileState {
entries
.iter()
.map(|(path, seconds, size)| {
(
PathBuf::from(path),
(
SystemTime::UNIX_EPOCH + Duration::from_secs(*seconds),
*size,
),
)
})
.collect()
}
#[test]
fn changed_artifacts_detects_new_and_modified_beams() {
let before = ArtifactState {
beams: file_state(&[("ebin/a.beam", 1, 10), ("ebin/b.beam", 1, 20)]),
nifs: FileState::new(),
};
let after = ArtifactState {
beams: file_state(&[
("ebin/a.beam", 2, 10),
("ebin/b.beam", 1, 20),
("ebin/c.beam", 1, 30),
]),
nifs: FileState::new(),
};
let diff = changed_artifacts(&before, &after);
assert_eq!(diff.modules, vec!["a".to_owned(), "c".to_owned()]);
assert!(!diff.nifs_changed);
}
#[test]
fn changed_artifacts_ignores_unchanged_state() {
let state = ArtifactState {
beams: file_state(&[("ebin/a.beam", 1, 10)]),
nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
};
let diff = changed_artifacts(&state, &state.clone());
assert!(diff.modules.is_empty());
assert!(!diff.nifs_changed);
}
#[test]
fn changed_artifacts_flags_nif_changes() {
let before = ArtifactState {
beams: FileState::new(),
nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
};
let after = ArtifactState {
beams: FileState::new(),
nifs: file_state(&[("priv/a_nif.so", 2, 11)]),
};
assert!(changed_artifacts(&before, &after).nifs_changed);
}
#[test]
fn reload_eval_quotes_nodes_and_modules() {
let eval = build_reload_eval(
&["fluxer_gateway_websocket_1@127.0.0.1".to_owned()],
&["gateway_compress".to_owned(), "push".to_owned()],
);
assert!(eval.contains("Nodes = ['fluxer_gateway_websocket_1@127.0.0.1']"));
assert!(eval.contains("Mods = ['gateway_compress','push']"));
assert!(eval.contains("halt(0)."));
}
#[test]
fn reload_outcome_parses_failed_nodes() {
let output = "gateway_reload_error n1 mod {error,nofile}\n\
gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\
gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\
gateway_reload_done\n";
let outcome = parse_reload_outcome(output);
assert_eq!(
outcome.failed_nodes,
vec!["fluxer_gateway_guilds_2@127.0.0.1".to_owned()]
);
}
#[test]
fn reload_outcome_empty_when_no_failures() {
assert!(
parse_reload_outcome("gateway_reload_done\n")
.failed_nodes
.is_empty()
);
}
#[test]
fn watched_source_filter_accepts_known_extensions() {
assert!(is_watched_source(Path::new("src/push.erl")));
assert!(is_watched_source(Path::new("include/gateway.hrl")));
assert!(is_watched_source(Path::new("src/fluxer_gateway.app.src")));
assert!(is_watched_source(Path::new("native/a_nif/src/lib.rs")));
assert!(is_watched_source(Path::new("native/a_nif/Cargo.toml")));
assert!(!is_watched_source(Path::new("ebin/push.beam")));
assert!(!is_watched_source(Path::new("src/.push.erl.swp")));
assert!(!is_watched_source(Path::new("native/a_nif/Cargo.lock")));
}
#[test]
fn skipped_dirs_cover_build_output_and_hidden_dirs() {
assert!(is_skipped_dir(Path::new("native/a_nif/target")));
assert!(is_skipped_dir(Path::new("src/.git")));
assert!(!is_skipped_dir(Path::new("src/gateway")));
}
}
tools/dev/src/lib.rs
-1
View File
@@ -7,7 +7,6 @@ pub mod dev;
pub mod disclaim;
pub mod env;
pub mod gateway;
pub mod gateway_reload;
pub mod manifest;
pub mod media_external;
pub mod media_proxy;
tools/dev/src/main.rs
+107 -7
View File
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::Result;
use anyhow::{Context, Result, bail};
use clap::{Args, Parser, Subcommand};
use fluxer_dev::cassandra::{
apply_schema, compute_diff, render_target_schema, verify_schema, write_diff_file,
@@ -11,9 +11,19 @@ use fluxer_dev::desktop::{
};
use fluxer_dev::env::merge_default_env_with_current;
use fluxer_dev::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL};
use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT_LOCAL_ENV_FILE};
use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::process::Command as ProcessCommand;
const DEV_INFRA_SERVICES: &[&str] = &[
"postgres",
"valkey",
"nats",
"livekit",
"meilisearch",
"mailpit",
];
#[derive(Debug, Parser)]
#[command(name = "fluxer-dev")]
@@ -34,6 +44,7 @@ enum Command {
Proxy(ProxyArgs),
Dev(DevArgs),
RustServices(RustServicesArgs),
Infra(InfraArgs),
Cassandra(CassandraArgs),
Desktop(DesktopArgs),
MediaProxy(MediaProxyArgs),
@@ -44,8 +55,6 @@ enum Command {
struct BootstrapArgs {
#[arg(long)]
skip_install: bool,
#[arg(long)]
skip_desktop_install: bool,
}
#[derive(Debug, Args)]
@@ -76,6 +85,19 @@ struct RustServicesArgs {
services: Vec<String>,
}
#[derive(Debug, Args)]
struct InfraArgs {
#[command(subcommand)]
command: InfraCommand,
}
#[derive(Debug, Subcommand)]
enum InfraCommand {
Start,
Stop,
Status,
}
#[derive(Debug, Args)]
struct CassandraArgs {
#[command(subcommand)]
@@ -193,10 +215,12 @@ async fn main() -> Result<()> {
match cli.command {
Command::Bootstrap(args) => {
fluxer_dev::bootstrap::bootstrap(args.skip_install, args.skip_desktop_install).await?;
fluxer_dev::bootstrap::bootstrap(args.skip_install).await?;
}
Command::PostStart => fluxer_dev::bootstrap::post_start().await?,
Command::Gateway(args) if args.mode == "single" => fluxer_dev::gateway::run_gateway()?,
Command::Gateway(args) if args.mode == "single" => {
std::process::exit(fluxer_dev::gateway::run_gateway().await?)
}
Command::Gateway(_) => {
std::process::exit(fluxer_dev::gateway::run_gateway_cluster().await?)
}
@@ -216,6 +240,7 @@ async fn main() -> Result<()> {
Command::RustServices(args) => {
std::process::exit(fluxer_dev::rust_services::run_rust_services(&args.services).await?)
}
Command::Infra(args) => run_infra(args.command)?,
Command::Cassandra(args) => match args.command {
CassandraCommand::Diff { output } => {
let diff = compute_diff(None).await?;
@@ -315,3 +340,78 @@ fn apply_default_env() -> Result<()> {
}
Ok(())
}
fn run_infra(command: InfraCommand) -> Result<()> {
let project = compose_project_name()?;
let compose_file = ROOT.join(".devcontainer/docker-compose.yml");
let mut args = vec![
"compose".to_owned(),
"--project-name".to_owned(),
project,
"-f".to_owned(),
compose_file.display().to_string(),
];
match command {
InfraCommand::Start => args.push("start".to_owned()),
InfraCommand::Stop => args.push("stop".to_owned()),
InfraCommand::Status => {
args.push("ps".to_owned());
args.push("--all".to_owned());
}
}
args.extend(
DEV_INFRA_SERVICES
.iter()
.map(|service| (*service).to_owned()),
);
let status = ProcessCommand::new("docker")
.args(&args)
.status()
.context("failed to run Docker Compose for the dev infrastructure")?;
if !status.success() {
bail!("Docker Compose dev infrastructure command failed with {status}");
}
Ok(())
}
fn compose_project_name() -> Result<String> {
if Path::new("/.dockerenv").exists() {
let container = std::fs::read_to_string("/etc/hostname")
.context("failed to read the current devcontainer hostname")?;
let container = container.trim();
if container.is_empty()
|| !container
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-'))
{
bail!("Current devcontainer hostname is invalid");
}
let output = ProcessCommand::new("docker")
.args([
"inspect",
"--format",
"{{ index .Config.Labels \"com.docker.compose.project\" }}",
container,
])
.output()
.context("failed to inspect the current devcontainer Compose project")?;
if !output.status.success() {
bail!(
"Could not discover the current devcontainer Compose project: {}",
String::from_utf8_lossy(&output.stderr).trim()
);
}
let project = String::from_utf8(output.stdout)
.context("devcontainer Compose project label is not valid UTF-8")?
.trim()
.to_owned();
if project.is_empty() || project == "<no value>" {
bail!("Current container has no com.docker.compose.project label");
}
return Ok(project);
}
std::env::var("COMPOSE_PROJECT_NAME")
.ok()
.filter(|project| !project.trim().is_empty())
.context("COMPOSE_PROJECT_NAME must be set when managing dev infrastructure from the host")
}
tools/dev/src/manifest.rs
+1
View File
@@ -7,6 +7,7 @@ pub const LOOPBACK_HOST: &str = "127.0.0.1";
pub const ANY_HOST: &str = "0.0.0.0";
pub const DEV_PROXY_PORT: u16 = 8088;
pub const DEV_PROXY_GATEWAY_PORTS_ENV: &str = "FLUXER_DEV_PROXY_GATEWAY_PORTS";
pub const APP_PORT: u16 = 3000;
pub const APP_PROXY_PORT: u16 = 8773;
pub const ADMIN_PORT: u16 = 3020;
tools/dev/src/media_proxy.rs
+70 -21
View File
@@ -17,6 +17,8 @@ const DEV_S3_ACCESS_KEY_ID: &str = "fluxer";
const DEV_S3_SECRET_ACCESS_KEY: &str = "fluxer-secret";
const DEV_S3_HOST: &str = "127.0.0.1";
const DEV_S3_PORT: u16 = 8333;
const DEV_SEAWEEDFS_STOP_TIMEOUT: Duration = Duration::from_secs(15);
const DEV_SEAWEEDFS_KILL_TIMEOUT: Duration = Duration::from_secs(2);
pub async fn run_dev_media_doctor(
repair: bool,
@@ -35,10 +37,20 @@ pub async fn run_dev_media_doctor(
}
pub async fn ensure_dev_object_store(repair: bool, repair_timeout_secs: u64) -> Result<()> {
let managed_process_running =
read_dev_seaweedfs_pid()?.is_some_and(managed_seaweedfs_process_running);
if repair && !tcp_reachable(DEV_S3_HOST, DEV_S3_PORT) && !managed_process_running {
println!("SeaweedFS S3 is not running; starting the managed development instance.");
start_dev_seaweedfs()?;
wait_s3_api(repair_timeout_secs).await?;
ensure_s3_buckets()?;
return Ok(());
}
match wait_s3_api(5).await {
Ok(()) => {}
Err(error) if repair => {
println!("SeaweedFS S3 check failed: {error}");
Err(_) if repair => {
println!("SeaweedFS S3 is unresponsive; restarting the managed development instance.");
start_dev_seaweedfs()?;
wait_s3_api(repair_timeout_secs).await?;
}
@@ -128,7 +140,7 @@ fn start_dev_seaweedfs() -> Result<()> {
if let Some(pid) = read_dev_seaweedfs_pid()? {
if managed_seaweedfs_process_running(pid) {
eprintln!("Stopping unresponsive managed SeaweedFS process {pid}.");
stop_managed_seaweedfs_process(pid);
stop_managed_seaweedfs_process(pid)?;
}
let _ = fs::remove_file(DEV_SEAWEEDFS_PID_FILE.as_path());
}
@@ -141,15 +153,24 @@ fn start_dev_seaweedfs() -> Result<()> {
let log_path = DEV_LOG_DIR.join("seaweedfs.log");
let log = OpenOptions::new()
.create(true)
.append(true)
.write(true)
.truncate(true)
.open(&log_path)
.with_context(|| format!("failed to open SeaweedFS log {}", log_path.display()))?;
let stderr = log
.try_clone()
.context("failed to clone SeaweedFS log handle")?;
let data_dir_arg = format!("-dir={}", DEV_SEAWEEDFS_DIR.display());
let child = Command::new(weed)
.args(["-logtostderr=true", "mini", &data_dir_arg])
let mut child = Command::new(weed)
.args([
"-logtostderr=true",
"mini",
&data_dir_arg,
"-admin.ui=false",
"-filer.disableDirListing",
"-s3.port.iceberg=0",
"-webdav=false",
])
.env("AWS_ACCESS_KEY_ID", DEV_S3_ACCESS_KEY_ID)
.env("AWS_SECRET_ACCESS_KEY", DEV_S3_SECRET_ACCESS_KEY)
.env("S3_BUCKET", DEV_S3_BUCKETS)
@@ -159,12 +180,16 @@ fn start_dev_seaweedfs() -> Result<()> {
.spawn()
.context("failed to start SeaweedFS")?;
let pid = child.id();
fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()).with_context(|| {
format!(
"failed to write SeaweedFS pid file {}",
DEV_SEAWEEDFS_PID_FILE.display()
)
})?;
if let Err(error) = fs::write(DEV_SEAWEEDFS_PID_FILE.as_path(), pid.to_string()) {
let _ = child.kill();
let _ = child.wait();
return Err(error).with_context(|| {
format!(
"failed to write SeaweedFS pid file {}",
DEV_SEAWEEDFS_PID_FILE.display()
)
});
}
println!(
"Started SeaweedFS dev object store with pid {pid}; logs: {}",
log_path.display()
@@ -195,20 +220,44 @@ fn managed_seaweedfs_process_running(pid: u32) -> bool {
}
#[cfg(unix)]
fn stop_managed_seaweedfs_process(pid: u32) {
unsafe {
libc::kill(pid as i32, libc::SIGTERM);
}
std::thread::sleep(Duration::from_secs(2));
if managed_seaweedfs_process_running(pid) {
unsafe {
libc::kill(pid as i32, libc::SIGKILL);
fn stop_managed_seaweedfs_process(pid: u32) -> Result<()> {
let result = unsafe { libc::kill(pid as i32, libc::SIGTERM) };
if result == -1 {
let error = std::io::Error::last_os_error();
if error.raw_os_error() != Some(libc::ESRCH) {
return Err(error).context("failed to terminate managed SeaweedFS process");
}
}
let terminate_deadline = Instant::now() + DEV_SEAWEEDFS_STOP_TIMEOUT;
while managed_seaweedfs_process_running(pid) && Instant::now() < terminate_deadline {
std::thread::sleep(Duration::from_millis(100));
}
if !managed_seaweedfs_process_running(pid) {
return Ok(());
}
let result = unsafe { libc::kill(pid as i32, libc::SIGKILL) };
if result == -1 {
let error = std::io::Error::last_os_error();
if error.raw_os_error() != Some(libc::ESRCH) {
return Err(error).context("failed to kill unresponsive managed SeaweedFS process");
}
}
let kill_deadline = Instant::now() + DEV_SEAWEEDFS_KILL_TIMEOUT;
while managed_seaweedfs_process_running(pid) && Instant::now() < kill_deadline {
std::thread::sleep(Duration::from_millis(100));
}
if managed_seaweedfs_process_running(pid) {
bail!("managed SeaweedFS process {pid} survived SIGKILL");
}
Ok(())
}
#[cfg(not(unix))]
fn stop_managed_seaweedfs_process(_pid: u32) {}
fn stop_managed_seaweedfs_process(_pid: u32) -> Result<()> {
Ok(())
}
async fn check_dev_media_path(base_url: &str, path: &str) -> Result<()> {
let path = if path.starts_with('/') {
tools/dev/src/object_store.rs
+2 -2
View File
@@ -168,11 +168,11 @@ fn s3_bucket_already_exists_output(output: &str) -> bool {
output.contains("BucketAlreadyExists") || output.contains("BucketAlreadyOwnedByYou")
}
pub async fn bootstrap_schema_and_object_store() -> Result<()> {
pub async fn bootstrap_schema() -> Result<()> {
if cassandra_backend_enabled() {
apply_schema(Some(config_from_env()?)).await?;
}
ensure_s3_buckets()
Ok(())
}
fn cassandra_backend_enabled() -> bool {
tools/dev/src/proc.rs
+210 -2
View File
@@ -4,9 +4,11 @@ use crate::env::merge_default_env_with_current;
use crate::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};
use anyhow::{Context, Result, bail};
use std::collections::{BTreeMap, VecDeque};
use std::future::Future;
use std::io::{BufRead, BufReader, Read};
use std::net::{TcpStream, ToSocketAddrs};
use std::path::Path;
use std::process::{Command, Output, Stdio};
use std::process::{Child, Command, Output, Stdio};
use std::time::{Duration, Instant};
use tokio::time::sleep;
@@ -172,6 +174,152 @@ pub fn run_command(args: &[&str], options: RunOptions<'_>) -> Result<Output> {
Ok(output)
}
pub async fn run_command_interruptible(
args: &[&str],
options: RunOptions<'_>,
shutdown: &mut ShutdownSignal,
) -> Result<AwaitOutcome<Output>> {
if options.capture {
bail!("interruptible commands do not support captured output");
}
println!("$ {}", format_command(args));
let env = merged_env(Some(&options.env), options.load_default_env)?;
let mut command = Command::new(args[0]);
command
.args(&args[1..])
.current_dir(options.cwd)
.env_clear()
.envs(env)
.stdin(Stdio::inherit())
.stdout(Stdio::inherit())
.stderr(Stdio::inherit());
configure_process_group(&mut command);
let mut child = command
.spawn()
.with_context(|| format!("failed to run {}", format_command(args)))?;
loop {
let status = match child.try_wait() {
Ok(status) => status,
Err(error) => {
stop_process_group(&mut child, Duration::from_secs(5)).await;
return Err(error)
.with_context(|| format!("failed to wait for {}", format_command(args)));
}
};
if let Some(status) = status {
stop_process_group(&mut child, Duration::from_secs(5)).await;
let output = Output {
status,
stdout: Vec::new(),
stderr: Vec::new(),
};
if options.check && !output.status.success() {
let code = output.status.code().unwrap_or(-1);
bail!(
"Command failed with exit code {code}: {}",
format_command(args)
);
}
return Ok(AwaitOutcome::Completed(output));
}
tokio::select! {
signal = shutdown.recv() => {
stop_process_group(&mut child, Duration::from_secs(5)).await;
return Ok(AwaitOutcome::Shutdown(signal));
}
_ = sleep(Duration::from_millis(100)) => {}
}
}
}
pub fn configure_process_group(command: &mut Command) {
#[cfg(unix)]
unsafe {
use std::os::unix::process::CommandExt;
command.pre_exec(|| {
if libc::setsid() == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
}
#[cfg(unix)]
pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> {
signal_process_group(process, libc::SIGTERM)
}
#[cfg(not(unix))]
pub fn terminate_process_group(process: &mut Child) -> std::io::Result<()> {
process.kill()
}
#[cfg(unix)]
pub fn force_kill_process_group(process: &mut Child) {
if let Err(error) = signal_process_group(process, libc::SIGKILL) {
eprintln!(
"Failed to send SIGKILL to process group {}: {error}",
process.id()
);
}
let _ = process.kill();
let _ = process.wait();
}
#[cfg(unix)]
pub fn process_group_running(process: &mut Child) -> bool {
if unsafe { libc::kill(-(process.id() as i32), 0) } == 0 {
return true;
}
let error = std::io::Error::last_os_error();
error.raw_os_error() != Some(libc::ESRCH)
}
#[cfg(not(unix))]
pub fn process_group_running(process: &mut Child) -> bool {
process.try_wait().ok().flatten().is_none()
}
#[cfg(not(unix))]
pub fn force_kill_process_group(process: &mut Child) {
let _ = process.kill();
let _ = process.wait();
}
#[cfg(unix)]
fn signal_process_group(process: &Child, signal: libc::c_int) -> std::io::Result<()> {
if unsafe { libc::kill(-(process.id() as i32), signal) } == 0 {
return Ok(());
}
let error = std::io::Error::last_os_error();
if error.raw_os_error() == Some(libc::ESRCH) {
return Ok(());
}
Err(error)
}
pub async fn stop_process_group(process: &mut Child, grace_period: Duration) {
if let Err(error) = terminate_process_group(process) {
eprintln!(
"Failed to terminate process group {}: {error}",
process.id()
);
}
let deadline = Instant::now() + grace_period;
loop {
let leader_exited = process.try_wait().ok().flatten().is_some();
if leader_exited && !process_group_running(process) {
return;
}
if Instant::now() >= deadline {
force_kill_process_group(process);
return;
}
sleep(Duration::from_millis(100)).await;
}
}
pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> Result<()> {
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_error = None;
@@ -199,6 +347,19 @@ pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> R
}
pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
wait_http_status(name, url, timeout_secs, |status| status.as_u16() < 500).await
}
pub async fn wait_http_success(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
wait_http_status(name, url, timeout_secs, |status| status.is_success()).await
}
async fn wait_http_status(
name: &str,
url: &str,
timeout_secs: u64,
accepts: impl Fn(reqwest::StatusCode) -> bool,
) -> Result<()> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
@@ -206,7 +367,7 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
let mut last_error = None;
while Instant::now() < deadline {
match client.get(url).send().await {
Ok(response) if response.status().as_u16() < 500 => {
Ok(response) if accepts(response.status()) => {
println!("{name} is reachable at {url}");
return Ok(());
}
@@ -221,6 +382,30 @@ pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
);
}
pub fn prefix_output(label: &str, reader: impl Read) {
let mut reader = BufReader::new(reader);
let mut line = Vec::new();
loop {
line.clear();
match reader.read_until(b'\n', &mut line) {
Ok(0) => return,
Ok(_) => {
if line.last() == Some(&b'\n') {
line.pop();
}
if line.last() == Some(&b'\r') {
line.pop();
}
println!("[{label}] {}", String::from_utf8_lossy(&line));
}
Err(error) => {
eprintln!("[{label}] output reader failed: {error}");
return;
}
}
}
}
pub const RESTART_WINDOW: Duration = Duration::from_secs(60);
pub const RESTART_LIMIT: usize = 5;
@@ -262,6 +447,29 @@ impl ShutdownSignal {
}
}
pub enum AwaitOutcome<T> {
Completed(T),
Shutdown(&'static str),
}
pub async fn await_or_shutdown<T, F>(shutdown: &mut ShutdownSignal, future: F) -> AwaitOutcome<T>
where
F: Future<Output = T>,
{
tokio::select! {
biased;
signal = shutdown.recv() => AwaitOutcome::Shutdown(signal),
output = future => AwaitOutcome::Completed(output),
}
}
pub async fn pending_shutdown(shutdown: &mut ShutdownSignal) -> Option<&'static str> {
match await_or_shutdown(shutdown, std::future::ready(())).await {
AwaitOutcome::Completed(()) => None,
AwaitOutcome::Shutdown(signal) => Some(signal),
}
}
#[cfg(not(unix))]
pub struct ShutdownSignal;
tools/dev/src/proxy.rs
+92 -14
View File
@@ -1,6 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::manifest::{ANY_HOST, DEV_PROXY_PORT, LOCAL_APP_URL, PROXY_ROUTES, ProxyRoute};
use crate::manifest::{
ANY_HOST, DEV_PROXY_GATEWAY_PORTS_ENV, DEV_PROXY_PORT, GATEWAY_PORT, LOCAL_APP_URL,
PROXY_ROUTES, ProxyRoute,
};
use anyhow::{Context, Result, bail};
use axum::{
Router,
@@ -14,7 +17,7 @@ use hyper_util::rt::TokioIo;
use std::collections::HashMap;
use std::env;
use std::net::SocketAddr;
use std::sync::{LazyLock, Mutex};
use std::sync::{Arc, LazyLock, Mutex};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
@@ -26,6 +29,7 @@ static ROUTE_CURSORS: LazyLock<Mutex<HashMap<&'static str, usize>>> =
#[derive(Clone)]
struct ProxyState {
http_client: reqwest::Client,
gateway_ports: Arc<[u16]>,
}
const BLOCKED_REQUEST_HEADERS: &[&str] = &[
@@ -67,9 +71,21 @@ pub async fn run_proxy(host: &str, port: u16) -> Result<()> {
.tcp_nodelay(true)
.build()
.context("failed to build dev proxy HTTP client")?;
let gateway_ports = gateway_proxy_ports()?;
println!(
"Fluxer dev proxy gateway ports: {}",
gateway_ports
.iter()
.map(u16::to_string)
.collect::<Vec<_>>()
.join(",")
);
let app = Router::new()
.fallback(any(proxy_request))
.with_state(ProxyState { http_client });
.with_state(ProxyState {
http_client,
gateway_ports,
});
axum::serve(
listener,
@@ -91,11 +107,29 @@ async fn proxy_request(
}
let route = route_for_path(&request_head.path);
let (target_host, target_port) = target_for_route(route, &state.gateway_ports);
if is_upgrade_request(&request_head) {
return proxy_upgrade(request, request_head, route, client_addr).await;
return proxy_upgrade(
request,
request_head,
route,
target_host,
target_port,
client_addr,
)
.await;
}
proxy_http(state, request, request_head, route, client_addr).await
proxy_http(
state,
request,
request_head,
route,
target_host,
target_port,
client_addr,
)
.await
}
async fn proxy_http(
@@ -103,9 +137,10 @@ async fn proxy_http(
request: Request<Body>,
request_head: RequestHead,
route: &'static ProxyRoute,
target_host: &'static str,
target_port: u16,
client_addr: SocketAddr,
) -> Response<Body> {
let (target_host, target_port) = target_for_route(route);
let target_url = upstream_http_url(&request_head.path, route, target_host, target_port);
let (parts, body) = request.into_parts();
let method = parts.method.clone();
@@ -155,10 +190,11 @@ async fn proxy_upgrade(
mut request: Request<Body>,
request_head: RequestHead,
route: &'static ProxyRoute,
target_host: &'static str,
target_port: u16,
client_addr: SocketAddr,
) -> Response<Body> {
let on_upgrade = hyper::upgrade::on(&mut request);
let (target_host, target_port) = target_for_route(route);
let mut target = match TcpStream::connect((target_host, target_port)).await {
Ok(target) => target,
Err(error) => return bad_gateway_response(error),
@@ -497,17 +533,59 @@ pub fn route_for_path(path: &str) -> &'static ProxyRoute {
PROXY_ROUTES.last().expect("proxy has fallback route")
}
pub fn target_for_route(route: &'static ProxyRoute) -> (&'static str, u16) {
if route.alternate_ports.is_empty() {
pub fn target_for_route(route: &'static ProxyRoute, gateway_ports: &[u16]) -> (&'static str, u16) {
let uses_gateway_ports = route.prefix == "/gateway";
let port_count = if uses_gateway_ports {
gateway_ports.len()
} else {
route.alternate_ports.len() + 1
};
assert!(
port_count > 0,
"proxy route must have at least one target port"
);
if port_count == 1 {
if uses_gateway_ports {
return (route.host, gateway_ports[0]);
}
return (route.host, route.port);
}
let ports = std::iter::once(route.port)
.chain(route.alternate_ports.iter().copied())
.collect::<Vec<_>>();
let mut cursors = ROUTE_CURSORS.lock().expect("route cursor lock poisoned");
let cursor = *cursors.get(route.prefix).unwrap_or(&0);
cursors.insert(route.prefix, (cursor + 1) % ports.len());
(route.host, ports[cursor])
cursors.insert(route.prefix, (cursor + 1) % port_count);
let port = if uses_gateway_ports {
gateway_ports[cursor]
} else if cursor == 0 {
route.port
} else {
route.alternate_ports[cursor - 1]
};
(route.host, port)
}
fn gateway_proxy_ports() -> Result<Arc<[u16]>> {
let raw = env::var(DEV_PROXY_GATEWAY_PORTS_ENV).unwrap_or_else(|_| GATEWAY_PORT.to_string());
let mut ports = Vec::new();
for token in raw.split(',') {
let token = token.trim();
if token.is_empty() {
bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains an empty port");
}
let port = token
.parse::<u16>()
.with_context(|| format!("Invalid port {token:?} in {DEV_PROXY_GATEWAY_PORTS_ENV}"))?;
if port == 0 {
bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} ports must be greater than zero");
}
if ports.contains(&port) {
bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} contains duplicate port {port}");
}
ports.push(port);
}
if ports.len() > 3 {
bail!("{DEV_PROXY_GATEWAY_PORTS_ENV} supports at most three ports");
}
Ok(ports.into())
}
pub fn rewrite_path(path: &str, route: &ProxyRoute) -> String {
tools/dev/src/rust_services.rs
+247 -134
View File
@@ -1,10 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::manifest::{DEV_PROXY_PORT, MEDIA_PROXY_PORT, RustServiceSpec, rust_services};
use crate::paths::ROOT;
use crate::paths::{ROOT, TARGET_DIR};
use crate::proc::{
RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,
restart_budget_exceeded, run_command,
AwaitOutcome, RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, await_or_shutdown,
configure_process_group, format_command, merged_env, pending_shutdown, prefix_output,
restart_budget_exceeded, run_command_interruptible,
};
use anyhow::{Result, bail};
use std::collections::{BTreeSet, VecDeque};
@@ -12,6 +13,8 @@ use std::env;
#[cfg(target_os = "linux")]
use std::fs;
use std::net::{SocketAddr, TcpListener};
#[cfg(target_os = "linux")]
use std::path::Path;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
use tokio::time::sleep;
@@ -34,26 +37,51 @@ impl SupervisedService {
pub async fn run_rust_services(service_names: &[String]) -> Result<i32> {
let selected = select_services(service_names)?;
cleanup_orphaned_service_processes(&selected).await?;
wait_for_service_ports_available(&selected)?;
build_services(&selected)?;
let mut shutdown = ShutdownSignal::new()?;
cleanup_orphaned_service_processes(&selected).await?;
if let Some(signal) = pending_shutdown(&mut shutdown).await {
println!("Received {signal}; stopping Rust service startup...");
return Ok(0);
}
wait_for_service_ports_available(&selected)?;
match build_services(&selected, &mut shutdown).await? {
AwaitOutcome::Completed(_) => {}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping Rust service startup...");
return Ok(0);
}
}
let mut supervised = Vec::new();
for spec in &selected {
for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {
let child = match start_service(spec, mode, port) {
Ok(child) => child,
Err(error) => {
stop_supervised_services(&mut supervised);
return Err(error);
}
};
supervised.push(SupervisedService {
spec: spec.clone(),
mode,
port,
child: start_service(spec, mode, port)?,
child,
restarts: VecDeque::new(),
});
}
}
loop {
if let Err(error) = restart_exited_services(&mut supervised).await {
stop_supervised_services(&mut supervised);
return Err(error);
match await_or_shutdown(&mut shutdown, restart_exited_services(&mut supervised)).await {
AwaitOutcome::Completed(Ok(())) => {}
AwaitOutcome::Completed(Err(error)) => {
stop_supervised_services(&mut supervised);
return Err(error);
}
AwaitOutcome::Shutdown(signal) => {
println!("Received {signal}; stopping Rust service tasks...");
stop_supervised_services(&mut supervised);
return Ok(0);
}
}
tokio::select! {
signal = shutdown.recv() => {
@@ -71,6 +99,7 @@ async fn restart_exited_services(supervised: &mut [SupervisedService]) -> Result
let Some(status) = entry.child.try_wait()? else {
continue;
};
crate::gateway::stop_child_processes(&mut [&mut entry.child]);
if restart_budget_exceeded(&mut entry.restarts, Instant::now()) {
bail!(
"Rust service {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",
@@ -145,7 +174,10 @@ pub fn select_services(service_names: &[String]) -> Result<Vec<RustServiceSpec>>
.collect())
}
fn build_services(services: &[RustServiceSpec]) -> Result<()> {
async fn build_services(
services: &[RustServiceSpec],
shutdown: &mut ShutdownSignal,
) -> Result<AwaitOutcome<std::process::Output>> {
let mut packages = Vec::new();
let mut seen = BTreeSet::new();
for spec in services {
@@ -156,7 +188,23 @@ fn build_services(services: &[RustServiceSpec]) -> Result<()> {
let mut args = vec!["cargo".to_owned(), "build".to_owned()];
args.extend(packages);
let refs = args.iter().map(String::as_str).collect::<Vec<_>>();
run_command(&refs, RunOptions::default()).map(drop)
let env = if env::var_os("CARGO_BUILD_JOBS").is_none() {
vec![(
"CARGO_BUILD_JOBS".to_owned(),
Some(env::var("FLUXER_DEV_CARGO_JOBS").unwrap_or_else(|_| "2".to_owned())),
)]
} else {
Vec::new()
};
run_command_interruptible(
&refs,
RunOptions {
env,
..RunOptions::default()
},
shutdown,
)
.await
}
fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child> {
@@ -172,14 +220,7 @@ fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child>
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
unsafe {
use std::os::unix::process::CommandExt;
command.pre_exec(|| {
libc::setsid();
Ok(())
});
}
configure_process_group(&mut command);
let mut child = command.spawn()?;
if let Some(stdout) = child.stdout.take() {
let stdout_label = label.clone();
@@ -212,11 +253,20 @@ pub fn service_command(spec: &RustServiceSpec) -> Vec<String> {
format!("run -p {}", spec.package),
];
}
let target_dir = env::var_os("CARGO_TARGET_DIR")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| TARGET_DIR.clone());
let target_dir = if target_dir.is_absolute() {
target_dir
} else {
ROOT.join(target_dir)
};
vec![
"cargo".to_owned(),
"run".to_owned(),
"-p".to_owned(),
spec.package.to_owned(),
target_dir
.join("debug")
.join(format!("{}{}", spec.package, std::env::consts::EXE_SUFFIX))
.display()
.to_string(),
]
}
@@ -238,18 +288,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String
.unwrap_or_else(|_| "nats://nats:4222".to_owned()),
),
),
(
"FLUXER_CASSANDRA_HOSTS".to_owned(),
Some(env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned())),
),
(
"FLUXER_CASSANDRA_KEYSPACE".to_owned(),
Some(env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned())),
),
(
"FLUXER_CASSANDRA_PORT".to_owned(),
Some(env::var("FLUXER_CASSANDRA_PORT").unwrap_or_else(|_| "9042".to_owned())),
),
];
if mode == "shard" {
envs.push(("FLUXER_SVC_SHARD_ID".to_owned(), Some("0".to_owned())));
@@ -284,13 +322,6 @@ pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String
envs
}
fn prefix_output(label: &str, reader: impl std::io::Read) {
use std::io::{BufRead, BufReader};
for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
println!("[{label}] {line}");
}
}
fn wait_for_service_ports_available(services: &[RustServiceSpec]) -> Result<()> {
let conflicts = services
.iter()
@@ -333,7 +364,7 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Res
"Stopping orphaned Rust service process group(s): {}",
format_pids(&pids)
);
let term_failed = signal_process_groups(&pids, libc::SIGTERM);
let term_failed = signal_process_groups(&leaders, libc::SIGTERM);
if !term_failed.is_empty() {
println!(
"SIGTERM delivery failed for orphaned Rust service pid(s): {}",
@@ -348,7 +379,12 @@ async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Res
return Ok(());
}
if Instant::now() >= deadline {
let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);
let remaining_leaders = leaders
.iter()
.filter(|leader| remaining.contains(&leader.pid))
.cloned()
.collect::<Vec<_>>();
let kill_failed = signal_process_groups(&remaining_leaders, libc::SIGKILL);
sleep(Duration::from_millis(200)).await;
let survivors = surviving_service_group_pids(&leaders)?;
if survivors.is_empty() {
@@ -380,19 +416,85 @@ async fn cleanup_orphaned_service_processes(_services: &[RustServiceSpec]) -> Re
}
#[cfg(target_os = "linux")]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[derive(Clone, Debug, Eq, PartialEq)]
struct RustServiceLeader {
pid: i32,
members: Vec<RustServiceProcessIdentity>,
}
#[cfg(target_os = "linux")]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RustServiceProcessIdentity {
pid: i32,
starttime: u64,
}
#[cfg(target_os = "linux")]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RustServiceProcess {
pid: i32,
ppid: i32,
pgid: i32,
state: char,
starttime: u64,
}
#[cfg(target_os = "linux")]
fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServiceLeader>> {
let binaries = services
let current_exe = std::env::current_exe()?;
let processes = service_process_snapshot()?;
let mut owned_pids = BTreeSet::new();
for process in &processes {
if process.ppid != 1 || proc_stat_state_is_dead(process.state) {
continue;
}
let args = proc_cmdline(process.pid);
let orphaned_supervisor = cmdline_is_service_supervisor(&args, &current_exe);
let orphaned_service = proc_has_managed_service_environment(process.pid, services)
&& fs::read_link(format!("/proc/{}/cwd", process.pid))
.ok()
.as_deref()
== Some(ROOT.as_path());
if orphaned_supervisor || orphaned_service {
owned_pids.insert(process.pid);
}
}
loop {
let previous_count = owned_pids.len();
for process in &processes {
if owned_pids.contains(&process.ppid) {
owned_pids.insert(process.pid);
}
}
if owned_pids.len() == previous_count {
break;
}
}
let group_pids = processes
.iter()
.map(|spec| format!("target/debug/{}", spec.package))
.filter(|process| owned_pids.contains(&process.pid) && process.pgid > 1)
.map(|process| process.pgid)
.collect::<BTreeSet<_>>();
let mut leaders = Vec::new();
let leaders = group_pids
.into_iter()
.map(|pid| RustServiceLeader {
pid,
members: processes
.iter()
.filter(|process| owned_pids.contains(&process.pid) && process.pgid == pid)
.map(|process| RustServiceProcessIdentity {
pid: process.pid,
starttime: process.starttime,
})
.collect(),
})
.collect::<Vec<_>>();
Ok(leaders)
}
#[cfg(target_os = "linux")]
fn service_process_snapshot() -> Result<Vec<RustServiceProcess>> {
let mut processes = Vec::new();
for entry in fs::read_dir("/proc")? {
let Ok(entry) = entry else {
continue;
@@ -404,18 +506,69 @@ fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServ
else {
continue;
};
if proc_parent_pid(pid) != Some(1) {
let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
continue;
}
if !cmdline_has_service_binary(&proc_cmdline(pid), &binaries) {
};
let Some((state, ppid, pgid, starttime)) = parse_proc_stat_process(&stat) else {
continue;
}
if let Some(starttime) = proc_stat_starttime(pid) {
leaders.push(RustServiceLeader { pid, starttime });
}
};
processes.push(RustServiceProcess {
pid,
ppid,
pgid,
state,
starttime,
});
}
leaders.sort_unstable_by_key(|leader| leader.pid);
Ok(leaders)
Ok(processes)
}
#[cfg(target_os = "linux")]
fn cmdline_is_service_supervisor(args: &[String], current_exe: &Path) -> bool {
args.first().map(Path::new) == Some(current_exe)
&& args.get(1).map(String::as_str) == Some("rust-services")
}
#[cfg(target_os = "linux")]
fn proc_has_managed_service_environment(pid: i32, services: &[RustServiceSpec]) -> bool {
let environment_bytes = fs::read(format!("/proc/{pid}/environ")).unwrap_or_default();
let environment = environment_bytes
.split(|byte| *byte == 0)
.filter_map(|entry| {
let separator = entry.iter().position(|byte| *byte == b'=')?;
let (key, value) = entry.split_at(separator);
Some((key, &value[1..]))
})
.filter_map(|(key, value)| {
Some((
std::str::from_utf8(key).ok()?,
std::str::from_utf8(value).ok()?,
))
})
.collect::<std::collections::HashMap<_, _>>();
let Some(name) = environment.get("FLUXER_SVC_NAME") else {
return false;
};
let Some(mode) = environment.get("FLUXER_SVC_MODE") else {
return false;
};
let Some(port) = environment
.get("FLUXER_SVC_PORT")
.and_then(|port| port.parse::<u16>().ok())
else {
return false;
};
if environment.get("FLUXER_SVC_LISTEN_HOST") != Some(&"0.0.0.0") {
return false;
}
services.iter().any(|service| {
service.name == *name
&& match *mode {
"router" => port == service.port_base,
"shard" => port == service.port_base + 1,
_ => false,
}
})
}
#[cfg(target_os = "linux")]
@@ -429,7 +582,7 @@ fn format_pids(pids: &[i32]) -> String {
.join(", ")
}
#[cfg(any(target_os = "linux", test))]
#[cfg(test)]
fn cmdline_has_service_binary(args: &[String], binaries: &BTreeSet<String>) -> bool {
args.first()
.map(|arg| binaries.iter().any(|binary| arg.ends_with(binary)))
@@ -446,30 +599,6 @@ fn proc_cmdline(pid: i32) -> Vec<String> {
.collect()
}
#[cfg(target_os = "linux")]
fn proc_parent_pid(pid: i32) -> Option<i32> {
fs::read_to_string(format!("/proc/{pid}/status"))
.ok()?
.lines()
.find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())
}
#[cfg(target_os = "linux")]
fn process_exists(pid: i32) -> bool {
assert!(pid > 0);
match proc_stat_state_and_pgid(pid) {
Some((state, _pgid)) => !proc_stat_state_is_dead(state),
None => false,
}
}
#[cfg(target_os = "linux")]
fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {
assert!(pid > 0);
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
parse_proc_stat_state_and_pgid(&stat)
}
#[cfg(any(target_os = "linux", test))]
fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
let (_, after_comm) = stat.rsplit_once(')')?;
@@ -481,10 +610,13 @@ fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
}
#[cfg(target_os = "linux")]
fn proc_stat_starttime(pid: i32) -> Option<u64> {
assert!(pid > 0);
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
parse_proc_stat_starttime(&stat)
fn parse_proc_stat_process(stat: &str) -> Option<(char, i32, i32, u64)> {
let (_, after_comm) = stat.rsplit_once(')')?;
let mut fields = after_comm.split_ascii_whitespace();
let state = fields.next()?.chars().next()?;
let ppid = fields.next()?.parse().ok()?;
let pgid = fields.next()?.parse().ok()?;
Some((state, ppid, pgid, parse_proc_stat_starttime(stat)?))
}
#[cfg(any(target_os = "linux", test))]
@@ -500,62 +632,43 @@ fn proc_stat_state_is_dead(state: char) -> bool {
#[cfg(target_os = "linux")]
fn surviving_service_group_pids(leaders: &[RustServiceLeader]) -> Result<Vec<i32>> {
let leader_pids = leaders
Ok(leaders
.iter()
.filter(|leader| service_group_has_owned_member(leader))
.map(|leader| leader.pid)
.collect::<std::collections::HashSet<_>>();
let mut survivors = Vec::new();
for entry in fs::read_dir("/proc")? {
let Ok(entry) = entry else {
continue;
};
let Some(pid) = entry
.file_name()
.to_str()
.and_then(|name| name.parse::<i32>().ok())
else {
continue;
};
let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
continue;
};
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
continue;
};
if proc_stat_state_is_dead(state) {
continue;
}
if leader_pids.contains(&pgid) {
survivors.push(pid);
continue;
}
let leader = leaders.iter().find(|leader| leader.pid == pid);
if let Some(leader) = leader
&& parse_proc_stat_starttime(&stat) == Some(leader.starttime)
{
survivors.push(pid);
}
}
survivors.sort_unstable();
Ok(survivors)
.collect())
}
#[cfg(target_os = "linux")]
fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {
fn service_group_has_owned_member(leader: &RustServiceLeader) -> bool {
leader.members.iter().any(|member| {
let Ok(stat) = fs::read_to_string(format!("/proc/{}/stat", member.pid)) else {
return false;
};
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
return false;
};
!proc_stat_state_is_dead(state)
&& pgid == leader.pid
&& parse_proc_stat_starttime(&stat) == Some(member.starttime)
})
}
#[cfg(target_os = "linux")]
fn signal_process_groups(leaders: &[RustServiceLeader], signal: i32) -> Vec<i32> {
assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);
let mut failed = Vec::with_capacity(pids.len());
for pid in pids {
assert!(*pid > 0);
let group_result = unsafe { libc::kill(-pid, signal) };
let mut failed = Vec::with_capacity(leaders.len());
for leader in leaders {
assert!(leader.pid > 1);
if !service_group_has_owned_member(leader) {
continue;
}
let group_result = unsafe { libc::kill(-leader.pid, signal) };
if group_result == 0 {
continue;
}
let direct_result = unsafe { libc::kill(*pid, signal) };
if direct_result == 0 {
continue;
}
if process_exists(*pid) {
failed.push(*pid);
if service_group_has_owned_member(leader) {
failed.push(leader.pid);
}
}
failed
tools/dev/src/tunnel.rs
+11 -5
View File
@@ -253,6 +253,16 @@ pub async fn run_cloudflare_tunnel(
return Ok(status.code().unwrap_or(1));
}
if running_inside_devcontainer() {
let container = std::fs::read_to_string("/etc/hostname")
.context("failed to read the current devcontainer hostname")?;
let container = container.trim();
if container.is_empty()
|| !container
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-'))
{
bail!("Current devcontainer hostname is invalid");
}
let docker = docker_command();
let mut command = Command::new(&docker[0]);
command.args(&docker[1..]);
@@ -261,11 +271,7 @@ pub async fn run_cloudflare_tunnel(
"run",
"--rm",
"--network",
&format!(
"container:{}",
std::env::var("HOSTNAME")
.unwrap_or_else(|_| "fluxer-dev-workspace-1".to_owned())
),
&format!("container:{container}"),
"cloudflare/cloudflared:latest",
"tunnel",
"run",