perf(repo): parallelise api tests and cache image builds in ghcr (#1984)

This commit is contained in:
Hampus
2026-08-24 21:24:39 +02:00
committed by GitHub
parent 3afad20e36
commit c226eb7211
38 changed files with 701 additions and 264 deletions
-5
View File
@@ -1,10 +1,5 @@
self-hosted-runner:
labels:
- blacksmith-4vcpu-ubuntu-2404
- blacksmith-4vcpu-ubuntu-2404-arm
- blacksmith-32vcpu-ubuntu-2404
- blacksmith-32vcpu-ubuntu-2404-arm
- blacksmith-32vcpu-windows-2025
- fluxer-desktop-macos-arm64
config-variables: null
+6 -6
View File
@@ -80,7 +80,7 @@ jobs:
GH_TOKEN: ${{ steps.create-token.outputs.token }}
FLUXER_BUILD_VERSION: ${{ inputs['build-version'] }}
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- resolve-calver
tools/ci/run.sh resolve-calver
--github-output
build:
@@ -97,9 +97,9 @@ jobs:
matrix:
include:
- platform: amd64
runner: blacksmith-4vcpu-ubuntu-2404
runner: ubuntu-24.04
- platform: arm64
runner: blacksmith-4vcpu-ubuntu-2404-arm
runner: ubuntu-24.04-arm
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
@@ -121,8 +121,8 @@ jobs:
build-args: |
BUILD_VERSION=${{ needs.meta.outputs.build_version }}
${{ inputs.extra-build-args }}
cache-from: type=gha,scope=${{ inputs.image }}-${{ matrix.platform }}
cache-to: type=gha,scope=${{ inputs.image }}-${{ matrix.platform }},mode=max,ignore-error=true
cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}:buildcache-${{ matrix.platform }}
cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}:buildcache-${{ matrix.platform }},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
env:
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
@@ -176,7 +176,7 @@ jobs:
VERSION: ${{ needs.meta.outputs.build_version }}
RELEASE_BASELINE_SHA: ${{ vars.RELEASE_BASELINE_SHA }}
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- release
tools/ci/run.sh release
publish
--component "${{ inputs.image }}"
--build-version "${VERSION}"
+13 -13
View File
@@ -53,14 +53,14 @@ jobs:
- name: set variables
id: vars
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy
tools/ci/run.sh build-app-proxy
--step set_metadata
--build-version "${{ inputs['build-version'] }}"
build:
name: build app-proxy (amd64)
needs: meta
runs-on: blacksmith-4vcpu-ubuntu-2404
runs-on: ubuntu-24.04
timeout-minutes: 45
permissions:
actions: read
@@ -76,7 +76,7 @@ jobs:
toolchain: "1.93.0"
- name: prepare docker config
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy
tools/ci/run.sh build-app-proxy
--step prepare_docker_config
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- name: configure ghcr auth
@@ -84,24 +84,24 @@ jobs:
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy
tools/ci/run.sh build-app-proxy
--step configure_ghcr_auth
- name: build and push image + extract assets
env:
BUILD_VERSION: ${{ needs.meta.outputs.build_version }}
PUBLIC_ASSET_BASE_URL: https://fluxerstatic.com
CACHE_FROM: type=gha,scope=fluxer-app-proxy
CACHE_TO: type=gha,scope=fluxer-app-proxy,mode=max
CACHE_FROM: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64
CACHE_TO: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy
tools/ci/run.sh build-app-proxy
--step build_and_extract
- name: generate asset manifest
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy
tools/ci/run.sh build-app-proxy
--step generate_asset_manifest
- name: upload assets to S3 static bucket
@@ -111,13 +111,13 @@ jobs:
S3_ENDPOINT: ${{ vars.STATIC_S3_ENDPOINT }}
STATIC_BUCKET: ${{ vars.STATIC_S3_BUCKET }}
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy
tools/ci/run.sh build-app-proxy
--step upload_assets
build-arm64:
name: build app-proxy (arm64)
needs: meta
runs-on: blacksmith-4vcpu-ubuntu-2404-arm
runs-on: ubuntu-24.04-arm
timeout-minutes: 60
permissions:
actions: read
@@ -145,8 +145,8 @@ jobs:
BUILD_VERSION=${{ needs.meta.outputs.build_version }}
PUBLIC_ASSET_BASE_URL=https://fluxerstatic.com
BUNDLE_LOCAL_ASSETS=false
cache-from: type=gha,scope=fluxer-app-proxy-arm64
cache-to: type=gha,scope=fluxer-app-proxy-arm64,mode=max,ignore-error=true
cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-arm64
cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-arm64,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
env:
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
@@ -203,7 +203,7 @@ jobs:
VERSION: ${{ needs.meta.outputs.build_version }}
RELEASE_BASELINE_SHA: ${{ vars.RELEASE_BASELINE_SHA }}
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- release
tools/ci/run.sh release
publish
--component fluxer-app-proxy
--build-version "${VERSION}"
+2 -2
View File
@@ -119,7 +119,7 @@ jobs:
- matrix
runs-on: ${{ matrix.os }}
environment: desktop-releases
timeout-minutes: 60
timeout-minutes: 180
permissions:
actions: read
contents: read
@@ -508,7 +508,7 @@ jobs:
- build
runs-on: ubuntu-24.04-arm
environment: desktop-releases
timeout-minutes: 60
timeout-minutes: 180
permissions:
contents: read
env:
+1 -1
View File
@@ -39,7 +39,7 @@ jobs:
- name: Install dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
tools/ci/run.sh ci
--step install_dependencies
- name: Generate OpenAPI schemas
+1 -1
View File
@@ -57,7 +57,7 @@ jobs:
- name: Install dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
tools/ci/run.sh ci
--step install_dependencies
- name: Refresh source catalogs
+1 -1
View File
@@ -63,7 +63,7 @@ jobs:
- name: Install dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
tools/ci/run.sh ci
--step install_dependencies
- name: Compile translated catalogs
+206 -87
View File
@@ -3,21 +3,29 @@ name: Tests
on:
pull_request:
push:
branches:
- main
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
GHCR_REGISTRY: ghcr.io/${{ github.repository_owner }}
CARGO_PROFILE_DEV_DEBUG: none
CARGO_PROFILE_TEST_DEBUG: none
CARGO_INCREMENTAL: '0'
jobs:
typecheck:
runs-on: ubuntu-24.04
timeout-minutes: 25
env:
FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
@@ -28,6 +36,28 @@ jobs:
toolchain: "1.93.0"
targets: wasm32-unknown-unknown
- name: Restore ci helper
id: ci-helper
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: target/debug/fluxer-ci
key: >-
fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml',
'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }}
- name: Build ci helper
if: steps.ci-helper.outputs.cache-hit != 'true'
run: cargo build --locked --package fluxer-ci
- name: Save ci helper
if: github.ref == 'refs/heads/main' && steps.ci-helper.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: target/debug/fluxer-ci
key: >-
fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml',
'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }}
- name: Install pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271
@@ -38,18 +68,19 @@ jobs:
cache: 'pnpm'
- name: Install dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step install_dependencies
run: |
"$FLUXER_CI_BIN" ci --step install_dependencies
- name: Run typecheck
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step typecheck
run: |
"$FLUXER_CI_BIN" ci --step typecheck
test:
runs-on: ubuntu-24.04
timeout-minutes: 25
env:
FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci
PNPM_TEST_WORKSPACE_CONCURRENCY: '2'
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
@@ -60,6 +91,19 @@ jobs:
toolchain: "1.93.0"
targets: wasm32-unknown-unknown
- name: Restore ci helper
id: ci-helper
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: target/debug/fluxer-ci
key: >-
fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml',
'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }}
- name: Build ci helper
if: steps.ci-helper.outputs.cache-hit != 'true'
run: cargo build --locked --package fluxer-ci
- name: Install pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271
@@ -70,17 +114,45 @@ jobs:
cache: 'pnpm'
- name: Install dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step install_dependencies
run: |
"$FLUXER_CI_BIN" ci --step install_dependencies
- name: Restore fluxer_app wasm artifacts
id: app-wasm
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
- name: Run tests
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step test
run: |
"$FLUXER_CI_BIN" ci --step test
- name: Save fluxer_app wasm artifacts
if: always() && github.ref == 'refs/heads/main' && steps.app-wasm.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
rust:
runs-on: blacksmith-4vcpu-ubuntu-2404
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- name: Checkout code
@@ -102,15 +174,12 @@ jobs:
cache: 'pnpm'
- name: Cache cargo
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: rust-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
restore-keys: |
rust-${{ runner.os }}-
workspaces: |
. -> target
fluxer_desktop/native/rust -> target
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install native dependencies
run: |
@@ -130,15 +199,23 @@ jobs:
- name: Check formatting
run: cargo fmt --all -- --check
- name: Check formatting (desktop native)
run: cargo fmt --manifest-path fluxer_desktop/native/rust/Cargo.toml --all -- --check
- name: Clippy (warnings as errors)
run: cargo clippy --workspace -- -D warnings
- name: Run tests
run: cargo test --workspace
- name: Run desktop native tests
run: cargo test --manifest-path fluxer_desktop/native/rust/Cargo.toml
gateway:
runs-on: ubuntu-24.04
timeout-minutes: 25
env:
FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
@@ -148,45 +225,82 @@ jobs:
with:
toolchain: "1.93.0"
- name: Cache cargo (gateway NIFs)
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6
with:
workspaces: |
fluxer_gateway/native/guild_member_list_oset_nif -> target
fluxer_gateway/native/push_markdown_plaintext_nif -> target
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Restore ci helper
id: ci-helper
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: target/debug/fluxer-ci
key: >-
fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml',
'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }}
- name: Build ci helper
if: steps.ci-helper.outputs.cache-hit != 'true'
run: cargo build --locked --package fluxer-ci
- name: Set up Erlang
uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124
with:
otp-version: '28'
rebar3-version: '3.24.0'
- name: Cache rebar3 dependencies
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
- name: Restore rebar3 dependencies
id: rebar3-cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: |
fluxer_gateway/_build
~/.cache/rebar3
key: rebar3-${{ runner.os }}-${{ hashFiles('fluxer_gateway/rebar.lock') }}
fluxer_gateway/_build
!fluxer_gateway/_build/default/lib/fluxer_gateway
!fluxer_gateway/_build/test/lib/fluxer_gateway
key: >-
rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock',
'fluxer_gateway/rebar.config') }}
restore-keys: |
rebar3-${{ runner.os }}-
rebar3-${{ runner.os }}-otp28-rebar3.24.0-
- name: Check formatting
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step gateway_fmt
run: |
"$FLUXER_CI_BIN" ci --step gateway_fmt
- name: Compile
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step gateway_compile
run: |
"$FLUXER_CI_BIN" ci --step gateway_compile
- name: Run dialyzer
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step gateway_dialyzer
run: |
"$FLUXER_CI_BIN" ci --step gateway_dialyzer
- name: Run eunit tests
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step gateway_eunit
run: |
"$FLUXER_CI_BIN" ci --step gateway_eunit
- name: Save rebar3 dependencies
if: always() && github.ref == 'refs/heads/main' && steps.rebar3-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: |
~/.cache/rebar3
fluxer_gateway/_build
!fluxer_gateway/_build/default/lib/fluxer_gateway
!fluxer_gateway/_build/test/lib/fluxer_gateway
key: >-
rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock',
'fluxer_gateway/rebar.config') }}
knip:
runs-on: ubuntu-24.04
timeout-minutes: 25
env:
FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
@@ -197,6 +311,19 @@ jobs:
toolchain: "1.93.0"
targets: wasm32-unknown-unknown
- name: Restore ci helper
id: ci-helper
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: target/debug/fluxer-ci
key: >-
fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml',
'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }}
- name: Build ci helper
if: steps.ci-helper.outputs.cache-hit != 'true'
run: cargo build --locked --package fluxer-ci
- name: Install pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271
@@ -207,14 +334,42 @@ jobs:
cache: 'pnpm'
- name: Install dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step install_dependencies
run: |
"$FLUXER_CI_BIN" ci --step install_dependencies
- name: Restore fluxer_app wasm artifacts
id: app-wasm
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
- name: Run knip
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step knip
run: |
"$FLUXER_CI_BIN" ci --step knip
- name: Save fluxer_app wasm artifacts
if: always() && github.ref == 'refs/heads/main' && steps.app-wasm.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
i18n:
runs-on: ubuntu-24.04
@@ -223,12 +378,6 @@ jobs:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
- name: Set up Rust toolchain (CI helpers)
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
targets: wasm32-unknown-unknown
- name: Install pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271
@@ -239,9 +388,7 @@ jobs:
cache: 'pnpm'
- name: Install dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci
--step install_dependencies
run: pnpm install --frozen-lockfile
- name: Compile locale catalogs
run: pnpm i18n:compile
@@ -276,29 +423,6 @@ jobs:
- name: Verify shipped fonts match the lockfile
run: python3 tools/fonts/build_fonts.py --verify
ci-scripts:
runs-on: ubuntu-24.04
timeout-minutes: 25
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
components: rustfmt
- name: Sync ci helper dependencies
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci-scripts
--step sync
- name: Run ci helper tests
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci-scripts
--step test
helm-and-scripts:
runs-on: ubuntu-24.04
timeout-minutes: 10
@@ -306,19 +430,14 @@ jobs:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
- name: Set up Rust toolchain (CI helpers)
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
- name: Install helm
uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310
- name: Resolve Helm test build version
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- resolve-calver
--github-env
--env-name HELM_TEST_BUILD_VERSION
run: |
set -euo pipefail
read -r YEAR MONTH DAY TIME <<<"$(date -u '+%Y %m %d %H%M%S')"
echo "HELM_TEST_BUILD_VERSION=${YEAR}.$((10#$MONTH))${DAY}.$((10#$TIME))" >> "$GITHUB_ENV"
- name: Helm dependency update (all charts)
run: |
@@ -45,6 +45,10 @@ class SudoModeService {
let sudoModeServiceInstance: SudoModeService | null = null;
export function resetSudoModeServiceForTesting(): void {
sudoModeServiceInstance = null;
}
export function getSudoModeService(): SudoModeService {
if (!sudoModeServiceInstance) {
sudoModeServiceInstance = new SudoModeService();
@@ -25,6 +25,10 @@ interface NormalizedSsoConfigValidationResult extends SsoConfigValidationInput {
let ssoRequestUrlPolicy: RequestUrlPolicy | null = null;
export function resetSsoRequestUrlPolicyForTesting(): void {
ssoRequestUrlPolicy = null;
}
export function getSsoRequestUrlPolicy(): RequestUrlPolicy {
if (ssoRequestUrlPolicy === null) {
ssoRequestUrlPolicy = createPublicInternetRequestUrlPolicy({
@@ -174,6 +174,10 @@ export class LimitConfigService {
}
}
export function resetGlobalLimitConfigServiceForTesting(): void {
globalLimitConfigService = null;
}
export function getGlobalLimitConfigSnapshot(): LimitConfigSnapshot {
if (!globalLimitConfigService) {
throw new Error('LimitConfigService global instance has not been initialized');
@@ -89,6 +89,15 @@ class BannedAvatarHashCache {
return this.banned.size;
}
resetForTesting(): void {
this.shutdown();
this.banned = new Set();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -85,6 +85,15 @@ class FileShaCache {
return this.banned.size;
}
resetForTesting(): void {
this.shutdown();
this.banned = new Set();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -144,6 +144,17 @@ export class PhraseBlocklistCache {
return this.rawPhraseSet.size;
}
resetForTesting(): void {
this.shutdown();
this.rawPhrases = [];
this.rawPhraseSet = new Set();
this.rebuildMatchers();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -180,6 +180,15 @@ export class ProfileSubstringBlocklistCache {
return !!canonical && this.getMatchersForCheck(scope).some((matchers) => matchers.rawSet.has(canonical));
}
resetForTesting(): void {
this.shutdown();
this.byScope = new Map();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -763,3 +763,15 @@ export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) =>
ctx.set('ncmecSubmissionService', getNcmecSubmissionService());
await next();
});
export function resetServiceMiddlewareForTesting(): void {
shutdownReportService();
_inboundSmsChallengeService = null;
_registrationEventsRepository = null;
_riskAssessmentRepository = null;
_historicalOutcomeRepository = null;
_suspiciousIpRepository = null;
_ipInfoService = null;
_registrationRiskEvaluator = null;
_liveKitWebhookService = null;
}
@@ -306,3 +306,20 @@ export function getLiveKitServiceInstance(): ILiveKitService | null {
export function getVoiceRoomStoreInstance(): IVoiceRoomStore | null {
return voiceRoomStoreInstance;
}
export function resetServiceRegistryForTesting(): void {
_kvClient = null;
_snowflakeService = null;
_billingRepository = null;
_blueskyOAuthService = undefined;
_blueskyOAuthInitializationPromise = null;
_blueskyOAuthConfigSignature = null;
_blueskyOAuthInitializationSignature = null;
_disabledBlueskyOAuthService = undefined;
voiceTopology = null;
voiceAvailabilityService = null;
liveKitServiceInstance = null;
voiceRoomStoreInstance = null;
voiceConfigSubscriber = null;
voiceInitializationPromise = null;
}
@@ -97,7 +97,7 @@ import {
getSnowflakeService,
getWorkerService,
} from './ServiceRegistry';
import {singleton} from './Singleton';
import {clearSingletonsForTesting, singleton} from './Singleton';
export const getUserRepository = singleton(() => new UserRepository(getKVClient()));
export const getGuildRepository = singleton(() => new GuildRepository());
@@ -121,7 +121,10 @@ export const getPasswordChangeRepository = singleton(() => new PasswordChangeRep
const getUserContactChangeLogRepository = singleton(() => new UserContactChangeLogRepository());
export const getDonationRepository = singleton(() => new DonationRepository());
const getAdminApiKeyRepository = singleton(() => new AdminApiKeyRepository());
export const getInstanceConfigRepository = singleton(() => new InstanceConfigRepository(getKVClient()));
export const getInstanceConfigRepository = singleton(
() => new InstanceConfigRepository(getKVClient()),
(repository) => repository.shutdown(),
);
export const getGatewayRolloutConfigPublisher = singleton(
() =>
new GatewayRolloutConfigPublisher(
@@ -200,6 +203,7 @@ const getDownloadsStorageService: () => IStorageService = (() => {
export const getErrorI18nService = singleton(() => new ErrorI18nService());
export const getLimitConfigService = singleton(
() => new LimitConfigService(getInstanceConfigRepository(), getCacheService(), getKVClient()),
(service) => service.shutdown(),
);
export const getPurgeQueue: () => IPurgeQueue = singleton(() =>
Config.bunny.purgeEnabled ? new BunnyPurgeQueue(getKVClient()) : new NoopPurgeQueue(),
@@ -434,3 +438,20 @@ export async function initializeServiceSingletons(): Promise<void> {
}
await serviceSingletonInitializationPromise;
}
export function resetServiceSingletonsForTesting(): void {
activityTracker?.shutdown();
clearSingletonsForTesting();
_virusScanInitPromise = null;
serviceSingletonInitializationPromise = null;
bulkMessageDeletionQueue = null;
bulkMessageDeletionQueueClient = null;
premiumStateQueue = null;
premiumStateQueueClient = null;
activityTracker = null;
activityTrackerClient = null;
activityBuffer = null;
activityBufferClient = null;
accountDeletionQueue = null;
accountDeletionQueueClient = null;
}
+15 -1
View File
@@ -1,7 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export function singleton<T>(factory: () => T): () => T {
const singletonClearers: Array<() => void> = [];
export function singleton<T>(factory: () => T, teardown?: (instance: T) => void): () => T {
let instance: T | undefined;
singletonClearers.push(() => {
if (instance !== undefined) {
teardown?.(instance);
}
instance = undefined;
});
return () => {
if (instance === undefined) {
instance = factory();
@@ -9,3 +17,9 @@ export function singleton<T>(factory: () => T): () => T {
return instance;
};
}
export function clearSingletonsForTesting(): void {
for (const clear of singletonClearers) {
clear();
}
}
@@ -146,6 +146,17 @@ class UrlBlocklistCache {
};
}
resetForTesting(): void {
this.shutdown();
this.exactUrls = new Set();
this.blockedDomains = new Set();
this.kvClient = null;
this.storageService = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -26,6 +26,10 @@ const FETCH_APPLICATIONS_BY_IDS_CQL = Applications.selectCql({
let cachedAdminSecretHash: string | null = null;
export function resetAdminSecretHashForTesting(): void {
cachedAdminSecretHash = null;
}
async function getAdminSecretHash(): Promise<string | null> {
const secret = Config.admin.oauthClientSecret;
if (!secret) {
@@ -28,3 +28,7 @@ export function isIpBanExempt(ip: string | null | undefined): boolean {
const key = getSameIpDecisionKey(ip);
return key !== null && getExemptDecisionKeys().has(key);
}
export function resetIpBanExemptionsForTesting(): void {
exemptDecisionKeys = null;
}
@@ -42,6 +42,7 @@ import {NoopLogger} from './mocks/NoopLogger';
import {NoopUnfurlerService} from './mocks/NoopUnfurlerService';
import {NoopGatewayService} from './NoopGatewayService';
import {NoopWorkerService} from './NoopWorkerService';
import {resetServiceStateForTesting} from './ResetServiceState';
import {InMemorySearchProvider} from './search/InMemorySearchProvider';
import {TestMediaService} from './TestMediaService';
@@ -91,6 +92,7 @@ export async function createApiTestHarness(options: CreateApiTestHarnessOptions
const mockBlueskyOAuthService = new MockBlueskyOAuthService();
setInjectedBlueskyOAuthService(mockBlueskyOAuthService);
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
resetServiceStateForTesting();
const {
app,
initialize: initializeApp,
@@ -152,6 +154,7 @@ export async function createApiTestHarness(options: CreateApiTestHarnessOptions
setInjectedMediaService(new TestMediaService(fallbackStorageService));
setInjectedSearchProviderService(new NullSearchProvider());
setInjectedBlueskyOAuthService(new MockBlueskyOAuthService());
resetServiceStateForTesting();
resetApiServicesForTesting();
}
async function requestJson(params: {
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {resetSudoModeServiceForTesting} from '../auth/services/SudoModeService';
import {resetSsoRequestUrlPolicyForTesting} from '../instance/SsoConfigValidation';
import {resetGlobalLimitConfigServiceForTesting} from '../limits/LimitConfigService';
import {bannedAvatarHashCache} from '../middleware/BannedAvatarHashCache';
import {fileShaCache} from '../middleware/FileShaCache';
import {ipBanCache} from '../middleware/IpBanMiddleware';
import {phraseBlocklistCache} from '../middleware/PhraseBlocklistCache';
import {profileSubstringBlocklistCache} from '../middleware/ProfileSubstringBlocklistCache';
import {resetServiceMiddlewareForTesting} from '../middleware/ServiceMiddleware';
import {resetServiceRegistryForTesting} from '../middleware/ServiceRegistry';
import {resetServiceSingletonsForTesting} from '../middleware/ServiceSingletons';
import {torExitListCache} from '../middleware/TorExitListCache';
import {urlBlocklistCache} from '../middleware/UrlBlocklistCache';
import {resetAdminSecretHashForTesting} from '../oauth/repositories/ApplicationRepository';
import {resetIpBanExemptionsForTesting} from '../risk/IpBanExemptions';
import {setThemeCssMaxBytesForTesting} from '../theme/ThemeService';
export function resetServiceStateForTesting(): void {
resetServiceRegistryForTesting();
resetServiceSingletonsForTesting();
resetServiceMiddlewareForTesting();
resetIpBanExemptionsForTesting();
resetGlobalLimitConfigServiceForTesting();
resetSudoModeServiceForTesting();
resetSsoRequestUrlPolicyForTesting();
resetAdminSecretHashForTesting();
setThemeCssMaxBytesForTesting(undefined);
ipBanCache.shutdown();
ipBanCache.resetCaches();
torExitListCache.shutdown();
torExitListCache.clearForTesting();
urlBlocklistCache.resetForTesting();
fileShaCache.resetForTesting();
phraseBlocklistCache.resetForTesting();
bannedAvatarHashCache.resetForTesting();
profileSubstringBlocklistCache.resetForTesting();
}
+15 -2
View File
@@ -5,7 +5,20 @@ import {FileSizeTooLargeError} from '@fluxer/errors/src/domains/core/FileSizeToo
import {Config} from '../Config';
import type {IStorageService} from '../infrastructure/IStorageService';
const MAX_CSS_BYTES = 8 * 1024 * 1024;
export const THEME_CSS_MAX_BYTES = 8 * 1024 * 1024;
let themeCssMaxBytesOverride: number | undefined;
export function setThemeCssMaxBytesForTesting(bytes: number | undefined): void {
themeCssMaxBytesOverride = bytes;
}
export function resolveThemeCssMaxBytes(): number {
if (Config.dev.testModeEnabled && themeCssMaxBytesOverride !== undefined) {
return themeCssMaxBytesOverride;
}
return THEME_CSS_MAX_BYTES;
}
export class ThemeService {
constructor(private readonly storageService: IStorageService) {}
@@ -14,7 +27,7 @@ export class ThemeService {
id: string;
}> {
const cssBytes = Buffer.from(css, 'utf-8');
if (cssBytes.length > MAX_CSS_BYTES) {
if (cssBytes.length > resolveThemeCssMaxBytes()) {
throw new FileSizeTooLargeError();
}
const themeId = randomBytes(8).toString('hex');
@@ -2,15 +2,17 @@
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
import {createTestAccount} from '../../auth/tests/AuthTestUtils';
import {getConfig} from '../../Config';
import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';
import {HTTP_STATUS} from '../../test/TestConstants';
import {createBuilder} from '../../test/TestRequestBuilder';
import {resolveThemeCssMaxBytes, setThemeCssMaxBytesForTesting, THEME_CSS_MAX_BYTES} from '../ThemeService';
interface ThemeCreateResponse {
id: string;
}
const MAX_CSS_BYTES = 8 * 1024 * 1024;
const TEST_MAX_CSS_BYTES = 4096;
describe('Theme CSS size limits', () => {
let harness: ApiTestHarness;
@@ -19,22 +21,33 @@ describe('Theme CSS size limits', () => {
});
beforeEach(async () => {
await harness.reset();
setThemeCssMaxBytesForTesting(TEST_MAX_CSS_BYTES);
});
afterAll(async () => {
setThemeCssMaxBytesForTesting(undefined);
await harness?.shutdown();
});
it('rejects CSS that exceeds the 8MB limit', async () => {
const user = await createTestAccount(harness);
const oversizedCss = 'a'.repeat(MAX_CSS_BYTES + 1);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedCss})
.expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE')
.execute();
it('enforces the 8MB limit in production and only honours the override in test mode', () => {
const config = getConfig();
expect(THEME_CSS_MAX_BYTES).toBe(8 * 1024 * 1024);
expect(resolveThemeCssMaxBytes()).toBe(TEST_MAX_CSS_BYTES);
config.dev.testModeEnabled = false;
try {
expect(resolveThemeCssMaxBytes()).toBe(THEME_CSS_MAX_BYTES);
} finally {
config.dev.testModeEnabled = true;
}
setThemeCssMaxBytesForTesting(undefined);
try {
expect(resolveThemeCssMaxBytes()).toBe(THEME_CSS_MAX_BYTES);
} finally {
setThemeCssMaxBytesForTesting(TEST_MAX_CSS_BYTES);
}
});
it('accepts CSS at exactly the 8MB limit', async () => {
it('accepts CSS at exactly the real 8MB limit through the full HTTP stack', async () => {
setThemeCssMaxBytesForTesting(undefined);
const user = await createTestAccount(harness);
const maxCss = 'a'.repeat(MAX_CSS_BYTES);
const maxCss = 'a'.repeat(THEME_CSS_MAX_BYTES);
const theme = await createBuilder<ThemeCreateResponse>(harness, user.token)
.post('/users/@me/themes')
.body({css: maxCss})
@@ -42,9 +55,38 @@ describe('Theme CSS size limits', () => {
.execute();
expect(theme.id).toBeDefined();
});
it('accepts CSS just under the 8MB limit', async () => {
it('rejects CSS that exceeds the real 8MB limit through the full HTTP stack', async () => {
setThemeCssMaxBytesForTesting(undefined);
const user = await createTestAccount(harness);
const nearMaxCss = 'a'.repeat(MAX_CSS_BYTES - 1);
const oversizedCss = 'a'.repeat(THEME_CSS_MAX_BYTES + 1);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedCss})
.expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE')
.execute();
});
it('rejects CSS that exceeds the limit', async () => {
const user = await createTestAccount(harness);
const oversizedCss = 'a'.repeat(TEST_MAX_CSS_BYTES + 1);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedCss})
.expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE')
.execute();
});
it('accepts CSS at exactly the limit', async () => {
const user = await createTestAccount(harness);
const maxCss = 'a'.repeat(TEST_MAX_CSS_BYTES);
const theme = await createBuilder<ThemeCreateResponse>(harness, user.token)
.post('/users/@me/themes')
.body({css: maxCss})
.expect(HTTP_STATUS.CREATED)
.execute();
expect(theme.id).toBeDefined();
});
it('accepts CSS just under the limit', async () => {
const user = await createTestAccount(harness);
const nearMaxCss = 'a'.repeat(TEST_MAX_CSS_BYTES - 1);
const theme = await createBuilder<ThemeCreateResponse>(harness, user.token)
.post('/users/@me/themes')
.body({css: nearMaxCss})
@@ -52,12 +94,14 @@ describe('Theme CSS size limits', () => {
.execute();
expect(theme.id).toBeDefined();
});
it('rejects CSS exceeding limit with multibyte unicode characters', async () => {
it('rejects CSS whose byte length exceeds the limit even when the character count does not', async () => {
const user = await createTestAccount(harness);
const unicodeChar = '\u{1F600}';
const bytesPerChar = Buffer.from(unicodeChar, 'utf-8').length;
const charsNeeded = Math.ceil((MAX_CSS_BYTES + 1) / bytesPerChar);
const charsNeeded = Math.ceil((TEST_MAX_CSS_BYTES + 1) / bytesPerChar);
const oversizedUnicodeCss = unicodeChar.repeat(charsNeeded);
expect(oversizedUnicodeCss.length).toBeLessThanOrEqual(TEST_MAX_CSS_BYTES);
expect(Buffer.byteLength(oversizedUnicodeCss, 'utf-8')).toBeGreaterThan(TEST_MAX_CSS_BYTES);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedUnicodeCss})
+48 -20
View File
@@ -16,38 +16,44 @@ function parseParallelInteger(value: string | undefined, fallback: number): numb
}
function resolveDefaultParallelWorkers(): number {
const halfCoreCountMinusOne = Math.floor(availableParallelism() / 2) - 1;
return Math.max(2, halfCoreCountMinusOne);
return Math.max(2, availableParallelism() - 1);
}
const DEFAULT_PARALLEL_WORKERS = resolveDefaultParallelWorkers();
const configuredMaxWorkers = parseParallelInteger(process.env.API_TEST_MAX_WORKERS, DEFAULT_PARALLEL_WORKERS);
const configuredMaxConcurrency = parseParallelInteger(process.env.API_TEST_MAX_CONCURRENCY, configuredMaxWorkers);
const MODULE_REGISTRY_TEST_FILES = [
'src/api/gif/GifRequestCountry.test.ts',
'src/api/risk/__tests__/AccountPolicyService.test.ts',
];
const sharedExclude = [
...configDefaults.exclude,
'pkgs/**',
'../fluxer_desktop/**',
'**/target/**',
'**/*Integration.test.ts',
'**/*ExttestIntegration.test.ts',
];
const sharedTestConfig = {
globals: true,
environment: 'node' as const,
setupFiles: ['./src/api/test/Setup.ts'],
pool: 'threads' as const,
testTimeout: 40000,
hookTimeout: 20000,
maxConcurrency: configuredMaxConcurrency,
};
export default defineConfig({
root: process.cwd(),
plugins: [tsconfigPaths()],
cacheDir: './node_modules/.vitest',
test: {
globals: true,
environment: 'node',
setupFiles: ['./src/api/test/Setup.ts'],
include: ['src/**/*.{test,spec}.{ts,tsx}'],
exclude: [
...configDefaults.exclude,
'pkgs/**',
'../fluxer_desktop/**',
'**/target/**',
'**/*Integration.test.ts',
'**/*ExttestIntegration.test.ts',
],
pool: 'threads',
fileParallelism: true,
maxWorkers: configuredMaxWorkers,
maxConcurrency: configuredMaxConcurrency,
isolate: true,
testTimeout: 40000,
hookTimeout: 20000,
fileParallelism: true,
reporters: ['default', 'json'],
outputFile: './test-results.json',
coverage: {
@@ -63,5 +69,27 @@ export default defineConfig({
'**/TestHelpers.tsx',
],
},
projects: [
{
plugins: [tsconfigPaths()],
test: {
...sharedTestConfig,
name: 'api',
include: ['src/**/*.{test,spec}.{ts,tsx}'],
exclude: [...sharedExclude, ...MODULE_REGISTRY_TEST_FILES],
isolate: false,
},
},
{
plugins: [tsconfigPaths()],
test: {
...sharedTestConfig,
name: 'api-module-registry',
include: MODULE_REGISTRY_TEST_FILES,
exclude: sharedExclude,
isolate: true,
},
},
],
},
});
+2 -2
View File
@@ -16,7 +16,7 @@
},
"scripts": {
"build": "pnpm wasm:codegen && pnpm generate:colors && pnpm generate:message-layout && pnpm generate:theme-variables && pnpm generate:masks && pnpm generate:css-types && pnpm lingui:compile && rm -rf dist && rspack build --mode production && pnpm tsx scripts/build-sw.mjs",
"dev": "TOKIO_WORKER_THREADS=4 RAYON_NUM_THREADS=4 cargo run --manifest-path ../tools/ci/Cargo.toml -- app-dev-server",
"dev": "TOKIO_WORKER_THREADS=4 RAYON_NUM_THREADS=4 ../tools/ci/run.sh app-dev-server",
"test": "vitest run",
"bench:member-list": "vitest bench --run src/features/member/utils/MemberListRangeUtils.bench.ts",
"bench:gif-picker": "vitest bench --run src/features/channel/components/pickers/gif/GifPickerStateMachine.bench.ts src/features/channel/components/pickers/gif/GifPickerGridData.bench.ts src/features/channel/components/pickers/gif/GifPickerLoadingSkeletonGridLayout.bench.ts src/features/channel/components/pickers/shared/TileFlowSolver.bench.ts",
@@ -46,7 +46,7 @@
"wasm:bench:download-realmedia": "node scripts/BenchLibfluxcore.mjs --download-realmedia",
"wasm:bench:highres": "pnpm wasm:codegen && node scripts/BenchLibfluxcore.mjs --profile=highres",
"wasm:bench:realmedia": "pnpm wasm:codegen && node scripts/BenchLibfluxcore.mjs --profile=realmedia --offline",
"wasm:codegen": "cargo run --locked --manifest-path ../tools/ci/Cargo.toml -- build-app-wasm",
"wasm:codegen": "../tools/ci/run.sh build-app-wasm",
"wasm:codegen:simd": "FLUXCORE_WASM_SIMD=1 pnpm wasm:codegen"
},
"browserslist": [
+1 -1
View File
@@ -10,7 +10,7 @@
"main": "dist/main/index.js",
"scripts": {
"build": "node scripts/build.mjs",
"set-channel": "cargo run --manifest-path ../tools/ci/Cargo.toml -- build-desktop --step set_build_channel",
"set-channel": "../tools/ci/run.sh build-desktop --step set_build_channel",
"test": "cargo test --manifest-path native/rust/Cargo.toml",
"typecheck": "tsgo --noEmit"
},
+1 -4
View File
@@ -17,10 +17,7 @@
{rebar3_lint, {git, "https://github.com/project-fifo/rebar3_lint.git", {branch, "master"}}}
]}.
{pre_hooks, [
{compile,
"cargo run --locked --quiet --manifest-path ../tools/ci/Cargo.toml -- build-gateway-nifs --gateway-dir ."}
]}.
{pre_hooks, [{compile, "../tools/ci/run.sh build-gateway-nifs --gateway-dir ."}]}.
{overrides, [
{override, cowboy, [{deps, [cowlib, ranch]}]},
+1 -1
View File
@@ -11,7 +11,7 @@
"test:coverage": "vitest run --coverage",
"test:watch": "vitest",
"typecheck": "tsgo --noEmit",
"generate": "cd proto && buf generate && cargo run --manifest-path ../../../tools/ci/Cargo.toml -- clean-schema-generated-files --root ../src/gen && cd .. && pnpm exec biome check --write --unsafe --no-errors-on-unmatched src/gen"
"generate": "cd proto && buf generate && ../../../tools/ci/run.sh clean-schema-generated-files --root ../src/gen && cd .. && pnpm exec biome check --write --unsafe --no-errors-on-unmatched src/gen"
},
"dependencies": {
"@bufbuild/protobuf": "^2.12.0",
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -euo pipefail
if [ -x "${FLUXER_CI_BIN:-}" ]; then
exec "$FLUXER_CI_BIN" "$@"
fi
script_dir="$(cd "$(dirname "$0")" && pwd)"
exec cargo run --quiet --manifest-path "$script_dir/Cargo.toml" -- "$@"
+12 -4
View File
@@ -147,13 +147,16 @@ fn build_and_extract_command() -> Result<CommandSpec> {
)
.env(
"CACHE_FROM",
env::var("CACHE_FROM")
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy".to_string()),
env::var("CACHE_FROM").unwrap_or_else(|_| default_app_proxy_cache_ref()),
)
.env(
"CACHE_TO",
env::var("CACHE_TO")
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy,mode=max".to_string()),
env::var("CACHE_TO").unwrap_or_else(|_| {
format!(
"{},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true",
default_app_proxy_cache_ref()
)
}),
)
.env(
"DOCKER_BUILD_SUMMARY",
@@ -165,6 +168,11 @@ fn build_and_extract_command() -> Result<CommandSpec> {
))
}
fn default_app_proxy_cache_ref() -> String {
let owner = ghcr_owner().unwrap_or_else(|_| "fluxerapp".to_string());
format!("type=registry,ref=ghcr.io/{owner}/fluxer-app-proxy:buildcache-amd64")
}
fn ghcr_owner() -> Result<String> {
for key in ["GHCR_OWNER", "GITHUB_REPOSITORY_OWNER", "OWNER"] {
if let Ok(value) = env::var(key) {
+1 -2
View File
@@ -142,8 +142,7 @@ fn build_markdown_parser_wasm(app_dir: &Path) -> Result<()> {
let rust_source_dir = app_dir.join("../packages/markdown_parser/rust");
let bytes_path =
app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts");
let temp = TempDir::new().context("Failed to create source temp directory")?;
let target_dir = temp.path().join("target");
let target_dir = rust_source_dir.join("target");
run_command(apply_wasm_c_toolchain(
CommandSpec::new("cargo")
+110 -62
View File
@@ -6,6 +6,7 @@ use crate::gateway::{GatewayStep, run_gateway_step};
use anyhow::{Context, Result};
use clap::{Args, ValueEnum};
use std::env;
use std::ffi::OsString;
use std::path::{Path, PathBuf};
#[derive(Debug, Args, Clone)]
@@ -27,17 +28,10 @@ enum CiStep {
GatewayEunit,
}
#[derive(Debug, Args, Clone)]
pub struct CiScriptsArgs {
#[arg(long, value_enum)]
step: CiScriptsStep,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "snake_case")]
enum CiScriptsStep {
Sync,
Test,
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum AppWasm {
Build,
ReuseIfPresent,
}
pub async fn run_ci(args: CiArgs) -> Result<()> {
@@ -51,7 +45,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> {
CiStep::Typecheck => {
ensure_desktop_build_channel_file(&root)?;
run_generators(&root, true)?;
run_app_test_artifact_generators(&root)?;
run_app_test_artifact_generators(&root, AppWasm::Build)?;
run_command(
CommandSpec::new("pnpm")
.args(["-r", "--if-present", "typecheck"])
@@ -60,7 +54,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> {
}
CiStep::Test => {
run_generators(&root, false)?;
run_app_test_artifact_generators(&root)?;
run_app_test_artifact_generators(&root, AppWasm::ReuseIfPresent)?;
run_workspace_tests(&root)?;
run_command(with_test_env(
CommandSpec::new("pnpm")
@@ -69,7 +63,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> {
))
}
CiStep::Knip => {
run_app_test_artifact_generators(&root)?;
run_app_test_artifact_generators(&root, AppWasm::ReuseIfPresent)?;
ensure_desktop_build_channel_file(&root)?;
run_fluxer_app_script(&root, "i18n:compile")?;
run_command(
@@ -98,11 +92,43 @@ fn ensure_desktop_build_channel_file(root: &Path) -> Result<()> {
write_build_channel_file(&root.join("fluxer_desktop"), &channel)
}
fn run_app_test_artifact_generators(root: &Path) -> Result<()> {
run_fluxer_app_script(root, "wasm:codegen")?;
fn run_app_test_artifact_generators(root: &Path, wasm: AppWasm) -> Result<()> {
match (wasm, missing_app_wasm_artifact(root)) {
(AppWasm::ReuseIfPresent, None) => {
println!("Reusing restored fluxer_app wasm artifacts");
}
(AppWasm::ReuseIfPresent, Some(missing)) => {
println!(
"Rebuilding fluxer_app wasm artifacts: {} is missing",
missing.display()
);
run_fluxer_app_script(root, "wasm:codegen")?;
}
(AppWasm::Build, _) => run_fluxer_app_script(root, "wasm:codegen")?,
}
run_fluxer_app_script(root, "generate:masks")
}
fn app_wasm_artifacts(root: &Path) -> Vec<PathBuf> {
let app_dir = root.join("fluxer_app");
vec![
app_dir.join("pkgs/libfluxcore/libfluxcore.js"),
app_dir.join("pkgs/libfluxcore/libfluxcore.d.ts"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bindgen.js"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bindgen.d.ts"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bg.wasm"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bg.wasm.d.ts"),
app_dir.join("pkgs/libfluxcore/package.json"),
app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts"),
]
}
fn missing_app_wasm_artifact(root: &Path) -> Option<PathBuf> {
app_wasm_artifacts(root)
.into_iter()
.find(|path| !path.exists())
}
fn run_fluxer_app_script(root: &Path, script: &str) -> Result<()> {
run_command(
CommandSpec::new("pnpm")
@@ -111,27 +137,6 @@ fn run_fluxer_app_script(root: &Path, script: &str) -> Result<()> {
)
}
pub async fn run_ci_scripts(args: CiScriptsArgs) -> Result<()> {
let root = repo_root()?;
match args.step {
CiScriptsStep::Sync => run_command(
CommandSpec::new("cargo")
.args([
"fetch",
"--locked",
"--manifest-path",
"tools/ci/Cargo.toml",
])
.current_dir(root),
),
CiScriptsStep::Test => run_command(
CommandSpec::new("cargo")
.args(["test", "--locked", "--manifest-path", "tools/ci/Cargo.toml"])
.current_dir(root),
),
}
}
fn run_generators(root: &Path, for_typecheck: bool) -> Result<()> {
for command in generator_commands(for_typecheck) {
run_command(command.current_dir(root))?;
@@ -155,28 +160,41 @@ fn generator_commands(for_typecheck: bool) -> Vec<CommandSpec> {
commands
}
fn workspace_test_args(concurrency: Option<&str>) -> Vec<OsString> {
let mut args = vec![OsString::from("-r")];
if let Some(concurrency) = concurrency {
args.push(OsString::from(format!(
"--workspace-concurrency={concurrency}"
)));
}
args.extend(
[
"--filter",
"!fluxer_api",
"--filter",
"!fluxer",
"--filter",
"!fluxer_desktop",
"--if-present",
"test",
]
.into_iter()
.map(OsString::from),
);
args
}
fn run_workspace_tests(root: &Path) -> Result<()> {
let workspace_concurrency =
env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").unwrap_or_else(|_| "2".to_string());
let concurrency = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").ok();
run_command(with_test_env(
CommandSpec::new("pnpm")
.args([
"-r",
&format!("--workspace-concurrency={workspace_concurrency}"),
"--filter",
"!fluxer_api",
"--filter",
"!fluxer",
"--if-present",
"test",
])
.args(workspace_test_args(concurrency.as_deref()))
.current_dir(root),
))
}
fn with_test_env(spec: CommandSpec) -> CommandSpec {
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
let api_workers = env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| "2".to_string());
spec.env("FLUXER_NATS_URL", &nats_url)
.env(
"FLUXER_NATS_CORE_URL",
@@ -186,11 +204,6 @@ fn with_test_env(spec: CommandSpec) -> CommandSpec {
"FLUXER_NATS_JETSTREAM_URL",
env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone()),
)
.env("API_TEST_MAX_WORKERS", &api_workers)
.env(
"API_TEST_MAX_CONCURRENCY",
env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers),
)
}
fn default_test_nats_url() -> String {
@@ -211,7 +224,6 @@ fn repo_root() -> Result<PathBuf> {
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::OsString;
#[test]
fn generator_commands_include_i18n_types_only_for_typecheck() {
@@ -237,7 +249,7 @@ mod tests {
}
#[test]
fn with_test_env_sets_all_nats_urls_and_concurrency() {
fn with_test_env_sets_all_nats_urls_and_leaves_worker_counts_to_vitest() {
let spec = with_test_env(CommandSpec::new("pnpm"));
let env = spec
.env
@@ -257,13 +269,49 @@ mod tests {
env.get(&OsString::from("FLUXER_NATS_JETSTREAM_URL")),
Some(&default_nats_url)
);
assert_eq!(
env.get(&OsString::from("API_TEST_MAX_WORKERS")),
Some(&OsString::from("2"))
assert!(!env.contains_key(&OsString::from("API_TEST_MAX_WORKERS")));
assert!(!env.contains_key(&OsString::from("API_TEST_MAX_CONCURRENCY")));
}
#[test]
fn workspace_tests_exclude_desktop_and_leave_concurrency_to_pnpm() {
let args = workspace_test_args(None);
assert_eq!(args[0], OsString::from("-r"));
assert!(
!args
.iter()
.any(|arg| arg.to_string_lossy().starts_with("--workspace-concurrency"))
);
assert!(args.windows(2).any(|pair| pair
== [
OsString::from("--filter"),
OsString::from("!fluxer_desktop")
]));
}
#[test]
fn workspace_tests_forward_an_explicit_concurrency_override() {
let args = workspace_test_args(Some("4"));
assert_eq!(args[1], OsString::from("--workspace-concurrency=4"));
}
#[test]
fn missing_app_wasm_artifact_reports_the_first_absent_output() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path();
assert_eq!(
env.get(&OsString::from("API_TEST_MAX_CONCURRENCY")),
Some(&OsString::from("2"))
missing_app_wasm_artifact(root),
Some(root.join("fluxer_app/pkgs/libfluxcore/libfluxcore.js"))
);
for path in app_wasm_artifacts(root) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, "").unwrap();
}
assert_eq!(missing_app_wasm_artifact(root), None);
}
}
+10 -10
View File
@@ -122,14 +122,14 @@ const PLATFORMS: &[Platform] = &[
platform: "windows",
arch: "x64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-windows-2025",
os: "windows-2025",
electron_arch: "x64",
},
Platform {
platform: "windows",
arch: "arm64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-windows-2025",
os: "windows-2025",
electron_arch: "arm64",
},
Platform {
@@ -143,14 +143,14 @@ const PLATFORMS: &[Platform] = &[
platform: "linux",
arch: "x64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-ubuntu-2404",
os: "ubuntu-24.04",
electron_arch: "x64",
},
Platform {
platform: "linux",
arch: "arm64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-ubuntu-2404-arm",
os: "ubuntu-24.04-arm",
electron_arch: "arm64",
},
];
@@ -3520,9 +3520,9 @@ mod tests {
assert_eq!(
selected,
vec![
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404-arm\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04-arm\",\"electron_arch\":\"arm64\"}",
]
);
}
@@ -3560,9 +3560,9 @@ mod tests {
assert_eq!(
selected,
vec![
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404-arm\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04-arm\",\"electron_arch\":\"arm64\"}",
]
);
}
-2
View File
@@ -34,7 +34,6 @@ enum Command {
BuildDesktopNativeAddon(desktop_native::BuildDesktopNativeAddonArgs),
BuildGatewayNifs(gateway::BuildGatewayNifsArgs),
Ci(ci_workflow::CiArgs),
CiScripts(ci_workflow::CiScriptsArgs),
CleanSchemaGeneratedFiles(schema::CleanSchemaGeneratedFilesArgs),
Gateway(gateway::GatewayArgs),
Release(release::ReleaseArgs),
@@ -53,7 +52,6 @@ pub async fn run() -> Result<()> {
}
Command::BuildGatewayNifs(args) => gateway::run_build_gateway_nifs(args),
Command::Ci(args) => ci_workflow::run_ci(args).await,
Command::CiScripts(args) => ci_workflow::run_ci_scripts(args).await,
Command::CleanSchemaGeneratedFiles(args) => schema::run_clean_generated_files(args),
Command::Gateway(args) => gateway::run_gateway(args),
Command::Release(args) => release::run(args).await,
+22 -20
View File
@@ -6,9 +6,6 @@ use anyhow::Result;
use std::env;
use std::path::Path;
const DEFAULT_TEST_WORKSPACE_CONCURRENCY: &str = "2";
const DEFAULT_API_TEST_WORKERS: &str = "2";
fn task_run(args: &[&str]) -> Result<()> {
run_command(
args,
@@ -22,8 +19,6 @@ fn task_run(args: &[&str]) -> Result<()> {
fn test_env() -> Vec<(String, Option<String>)> {
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
let api_workers =
env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| DEFAULT_API_TEST_WORKERS.to_owned());
vec![
("FLUXER_NATS_URL".to_owned(), Some(nats_url.clone())),
(
@@ -32,12 +27,7 @@ fn test_env() -> Vec<(String, Option<String>)> {
),
(
"FLUXER_NATS_JETSTREAM_URL".to_owned(),
Some(env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone())),
),
("API_TEST_MAX_WORKERS".to_owned(), Some(api_workers.clone())),
(
"API_TEST_MAX_CONCURRENCY".to_owned(),
Some(env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers)),
Some(env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or(nats_url)),
),
]
}
@@ -68,27 +58,39 @@ pub fn run_typecheck() -> Result<i32> {
pub fn run_test() -> Result<i32> {
run_generators(false)?;
let workspace_concurrency = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY")
.unwrap_or_else(|_| DEFAULT_TEST_WORKSPACE_CONCURRENCY.to_owned());
let env = test_env();
run_command(
&[
"pnpm",
"-r",
&format!("--workspace-concurrency={workspace_concurrency}"),
let mut args = vec!["pnpm".to_owned(), "-r".to_owned()];
if let Ok(concurrency) = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY") {
args.push(format!("--workspace-concurrency={concurrency}"));
}
args.extend(
[
"--filter",
"!fluxer_api",
"--filter",
"!fluxer",
"--filter",
"!fluxer_desktop",
"--if-present",
"test",
],
]
.into_iter()
.map(str::to_owned),
);
let env = test_env();
run_command(
&args.iter().map(String::as_str).collect::<Vec<_>>(),
RunOptions {
env: env.clone(),
load_default_env: false,
..RunOptions::default()
},
)?;
task_run(&[
"cargo",
"test",
"--manifest-path",
"fluxer_desktop/native/rust/Cargo.toml",
])?;
run_command(
&["pnpm", "--filter", "fluxer_api", "test"],
RunOptions {