From c226eb7211a6eaf7eb32cdcf5635d0f57cc04a98 Mon Sep 17 00:00:00 2001 From: Hampus Date: Mon, 24 Aug 2026 21:24:39 +0200 Subject: [PATCH] perf(repo): parallelise api tests and cache image builds in ghcr (#1984) --- .github/actionlint.yaml | 5 - .github/workflows/_build-image.yaml | 12 +- .github/workflows/build-app-proxy.yaml | 26 +- .github/workflows/build-desktop.yaml | 4 +- .github/workflows/dart-sdk-validation.yaml | 2 +- .github/workflows/i18n-source-sync.yaml | 2 +- .github/workflows/i18n-weblate-pr.yaml | 2 +- .github/workflows/tests.yaml | 293 ++++++++++++------ .../src/api/auth/services/SudoModeService.ts | 4 + .../src/api/instance/SsoConfigValidation.ts | 4 + .../src/api/limits/LimitConfigService.ts | 4 + .../api/middleware/BannedAvatarHashCache.ts | 9 + fluxer_api/src/api/middleware/FileShaCache.ts | 9 + .../api/middleware/PhraseBlocklistCache.ts | 11 + .../ProfileSubstringBlocklistCache.ts | 9 + .../src/api/middleware/ServiceMiddleware.ts | 12 + .../src/api/middleware/ServiceRegistry.ts | 17 + .../src/api/middleware/ServiceSingletons.ts | 25 +- fluxer_api/src/api/middleware/Singleton.ts | 16 +- .../src/api/middleware/UrlBlocklistCache.ts | 11 + .../repositories/ApplicationRepository.ts | 4 + fluxer_api/src/api/risk/IpBanExemptions.ts | 4 + fluxer_api/src/api/test/ApiTestHarness.ts | 3 + fluxer_api/src/api/test/ResetServiceState.ts | 39 +++ fluxer_api/src/api/theme/ThemeService.ts | 17 +- .../api/theme/tests/ThemeCssSizeLimit.test.ts | 74 ++++- fluxer_api/vitest.config.ts | 68 ++-- fluxer_app/package.json | 4 +- fluxer_desktop/package.json | 2 +- fluxer_gateway/rebar.config | 5 +- packages/schema/package.json | 2 +- tools/ci/run.sh | 11 + tools/ci/src/app_proxy.rs | 16 +- tools/ci/src/app_wasm.rs | 3 +- tools/ci/src/ci_workflow.rs | 172 ++++++---- tools/ci/src/desktop.rs | 20 +- tools/ci/src/lib.rs | 2 - tools/dev/src/tasks.rs | 42 +-- 38 files changed, 701 insertions(+), 264 deletions(-) create mode 100644 fluxer_api/src/api/test/ResetServiceState.ts create mode 100755 tools/ci/run.sh diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 139b20435..c59b54cc3 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,10 +1,5 @@ self-hosted-runner: labels: - - blacksmith-4vcpu-ubuntu-2404 - - blacksmith-4vcpu-ubuntu-2404-arm - - blacksmith-32vcpu-ubuntu-2404 - - blacksmith-32vcpu-ubuntu-2404-arm - - blacksmith-32vcpu-windows-2025 - fluxer-desktop-macos-arm64 config-variables: null diff --git a/.github/workflows/_build-image.yaml b/.github/workflows/_build-image.yaml index 19acb8b37..81d7d6bec 100644 --- a/.github/workflows/_build-image.yaml +++ b/.github/workflows/_build-image.yaml @@ -80,7 +80,7 @@ jobs: GH_TOKEN: ${{ steps.create-token.outputs.token }} FLUXER_BUILD_VERSION: ${{ inputs['build-version'] }} run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- resolve-calver + tools/ci/run.sh resolve-calver --github-output build: @@ -97,9 +97,9 @@ jobs: matrix: include: - platform: amd64 - runner: blacksmith-4vcpu-ubuntu-2404 + runner: ubuntu-24.04 - platform: arm64 - runner: blacksmith-4vcpu-ubuntu-2404-arm + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 env: @@ -121,8 +121,8 @@ jobs: build-args: | BUILD_VERSION=${{ needs.meta.outputs.build_version }} ${{ inputs.extra-build-args }} - cache-from: type=gha,scope=${{ inputs.image }}-${{ matrix.platform }} - cache-to: type=gha,scope=${{ inputs.image }}-${{ matrix.platform }},mode=max,ignore-error=true + cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}:buildcache-${{ matrix.platform }} + cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}:buildcache-${{ matrix.platform }},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true env: DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false @@ -176,7 +176,7 @@ jobs: VERSION: ${{ needs.meta.outputs.build_version }} RELEASE_BASELINE_SHA: ${{ vars.RELEASE_BASELINE_SHA }} run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- release + tools/ci/run.sh release publish --component "${{ inputs.image }}" --build-version "${VERSION}" diff --git a/.github/workflows/build-app-proxy.yaml b/.github/workflows/build-app-proxy.yaml index 056e7ed59..0e002cb0b 100644 --- a/.github/workflows/build-app-proxy.yaml +++ b/.github/workflows/build-app-proxy.yaml @@ -53,14 +53,14 @@ jobs: - name: set variables id: vars run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy + tools/ci/run.sh build-app-proxy --step set_metadata --build-version "${{ inputs['build-version'] }}" build: name: build app-proxy (amd64) needs: meta - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-24.04 timeout-minutes: 45 permissions: actions: read @@ -76,7 +76,7 @@ jobs: toolchain: "1.93.0" - name: prepare docker config run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy + tools/ci/run.sh build-app-proxy --step prepare_docker_config - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 - name: configure ghcr auth @@ -84,24 +84,24 @@ jobs: GHCR_USERNAME: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy + tools/ci/run.sh build-app-proxy --step configure_ghcr_auth - name: build and push image + extract assets env: BUILD_VERSION: ${{ needs.meta.outputs.build_version }} PUBLIC_ASSET_BASE_URL: https://fluxerstatic.com - CACHE_FROM: type=gha,scope=fluxer-app-proxy - CACHE_TO: type=gha,scope=fluxer-app-proxy,mode=max + CACHE_FROM: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64 + CACHE_TO: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy + tools/ci/run.sh build-app-proxy --step build_and_extract - name: generate asset manifest run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy + tools/ci/run.sh build-app-proxy --step generate_asset_manifest - name: upload assets to S3 static bucket @@ -111,13 +111,13 @@ jobs: S3_ENDPOINT: ${{ vars.STATIC_S3_ENDPOINT }} STATIC_BUCKET: ${{ vars.STATIC_S3_BUCKET }} run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-app-proxy + tools/ci/run.sh build-app-proxy --step upload_assets build-arm64: name: build app-proxy (arm64) needs: meta - runs-on: blacksmith-4vcpu-ubuntu-2404-arm + runs-on: ubuntu-24.04-arm timeout-minutes: 60 permissions: actions: read @@ -145,8 +145,8 @@ jobs: BUILD_VERSION=${{ needs.meta.outputs.build_version }} PUBLIC_ASSET_BASE_URL=https://fluxerstatic.com BUNDLE_LOCAL_ASSETS=false - cache-from: type=gha,scope=fluxer-app-proxy-arm64 - cache-to: type=gha,scope=fluxer-app-proxy-arm64,mode=max,ignore-error=true + cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-arm64 + cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-arm64,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true env: DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false @@ -203,7 +203,7 @@ jobs: VERSION: ${{ needs.meta.outputs.build_version }} RELEASE_BASELINE_SHA: ${{ vars.RELEASE_BASELINE_SHA }} run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- release + tools/ci/run.sh release publish --component fluxer-app-proxy --build-version "${VERSION}" diff --git a/.github/workflows/build-desktop.yaml b/.github/workflows/build-desktop.yaml index 5e0bca185..e7ae4aa8a 100644 --- a/.github/workflows/build-desktop.yaml +++ b/.github/workflows/build-desktop.yaml @@ -119,7 +119,7 @@ jobs: - matrix runs-on: ${{ matrix.os }} environment: desktop-releases - timeout-minutes: 60 + timeout-minutes: 180 permissions: actions: read contents: read @@ -508,7 +508,7 @@ jobs: - build runs-on: ubuntu-24.04-arm environment: desktop-releases - timeout-minutes: 60 + timeout-minutes: 180 permissions: contents: read env: diff --git a/.github/workflows/dart-sdk-validation.yaml b/.github/workflows/dart-sdk-validation.yaml index 5fc7b23bf..32d8e79a5 100644 --- a/.github/workflows/dart-sdk-validation.yaml +++ b/.github/workflows/dart-sdk-validation.yaml @@ -39,7 +39,7 @@ jobs: - name: Install dependencies run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci + tools/ci/run.sh ci --step install_dependencies - name: Generate OpenAPI schemas diff --git a/.github/workflows/i18n-source-sync.yaml b/.github/workflows/i18n-source-sync.yaml index 522a35a40..7a012f555 100644 --- a/.github/workflows/i18n-source-sync.yaml +++ b/.github/workflows/i18n-source-sync.yaml @@ -57,7 +57,7 @@ jobs: - name: Install dependencies run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci + tools/ci/run.sh ci --step install_dependencies - name: Refresh source catalogs diff --git a/.github/workflows/i18n-weblate-pr.yaml b/.github/workflows/i18n-weblate-pr.yaml index c5422e693..141bfa1a8 100644 --- a/.github/workflows/i18n-weblate-pr.yaml +++ b/.github/workflows/i18n-weblate-pr.yaml @@ -63,7 +63,7 @@ jobs: - name: Install dependencies run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci + tools/ci/run.sh ci --step install_dependencies - name: Compile translated catalogs diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ce605b105..b91c32f9f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -3,21 +3,29 @@ name: Tests on: pull_request: + push: + branches: + - main permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: GHCR_REGISTRY: ghcr.io/${{ github.repository_owner }} + CARGO_PROFILE_DEV_DEBUG: none + CARGO_PROFILE_TEST_DEBUG: none + CARGO_INCREMENTAL: '0' jobs: typecheck: runs-on: ubuntu-24.04 timeout-minutes: 25 + env: + FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -28,6 +36,28 @@ jobs: toolchain: "1.93.0" targets: wasm32-unknown-unknown + - name: Restore ci helper + id: ci-helper + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/debug/fluxer-ci + key: >- + fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml', + 'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }} + + - name: Build ci helper + if: steps.ci-helper.outputs.cache-hit != 'true' + run: cargo build --locked --package fluxer-ci + + - name: Save ci helper + if: github.ref == 'refs/heads/main' && steps.ci-helper.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/debug/fluxer-ci + key: >- + fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml', + 'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }} + - name: Install pnpm uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 @@ -38,18 +68,19 @@ jobs: cache: 'pnpm' - name: Install dependencies - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step install_dependencies + run: | + "$FLUXER_CI_BIN" ci --step install_dependencies - name: Run typecheck - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step typecheck + run: | + "$FLUXER_CI_BIN" ci --step typecheck test: runs-on: ubuntu-24.04 timeout-minutes: 25 + env: + FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci + PNPM_TEST_WORKSPACE_CONCURRENCY: '2' steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -60,6 +91,19 @@ jobs: toolchain: "1.93.0" targets: wasm32-unknown-unknown + - name: Restore ci helper + id: ci-helper + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/debug/fluxer-ci + key: >- + fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml', + 'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }} + + - name: Build ci helper + if: steps.ci-helper.outputs.cache-hit != 'true' + run: cargo build --locked --package fluxer-ci + - name: Install pnpm uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 @@ -70,17 +114,45 @@ jobs: cache: 'pnpm' - name: Install dependencies - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step install_dependencies + run: | + "$FLUXER_CI_BIN" ci --step install_dependencies + + - name: Restore fluxer_app wasm artifacts + id: app-wasm + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + fluxer_app/pkgs/libfluxcore + fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts + key: >- + app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs', + 'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts', + 'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock', + 'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**', + 'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml', + 'packages/markdown_parser/rust/src/**') }} - name: Run tests - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step test + run: | + "$FLUXER_CI_BIN" ci --step test + + - name: Save fluxer_app wasm artifacts + if: always() && github.ref == 'refs/heads/main' && steps.app-wasm.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + fluxer_app/pkgs/libfluxcore + fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts + key: >- + app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs', + 'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts', + 'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock', + 'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**', + 'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml', + 'packages/markdown_parser/rust/src/**') }} rust: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - name: Checkout code @@ -102,15 +174,12 @@ jobs: cache: 'pnpm' - name: Cache cargo - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: rust-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} - restore-keys: | - rust-${{ runner.os }}- + workspaces: | + . -> target + fluxer_desktop/native/rust -> target + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install native dependencies run: | @@ -130,15 +199,23 @@ jobs: - name: Check formatting run: cargo fmt --all -- --check + - name: Check formatting (desktop native) + run: cargo fmt --manifest-path fluxer_desktop/native/rust/Cargo.toml --all -- --check + - name: Clippy (warnings as errors) run: cargo clippy --workspace -- -D warnings - name: Run tests run: cargo test --workspace + - name: Run desktop native tests + run: cargo test --manifest-path fluxer_desktop/native/rust/Cargo.toml + gateway: runs-on: ubuntu-24.04 timeout-minutes: 25 + env: + FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -148,45 +225,82 @@ jobs: with: toolchain: "1.93.0" + - name: Cache cargo (gateway NIFs) + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 + with: + workspaces: | + fluxer_gateway/native/guild_member_list_oset_nif -> target + fluxer_gateway/native/push_markdown_plaintext_nif -> target + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Restore ci helper + id: ci-helper + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/debug/fluxer-ci + key: >- + fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml', + 'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }} + + - name: Build ci helper + if: steps.ci-helper.outputs.cache-hit != 'true' + run: cargo build --locked --package fluxer-ci + - name: Set up Erlang uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 with: otp-version: '28' rebar3-version: '3.24.0' - - name: Cache rebar3 dependencies - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + - name: Restore rebar3 dependencies + id: rebar3-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 with: path: | - fluxer_gateway/_build ~/.cache/rebar3 - key: rebar3-${{ runner.os }}-${{ hashFiles('fluxer_gateway/rebar.lock') }} + fluxer_gateway/_build + !fluxer_gateway/_build/default/lib/fluxer_gateway + !fluxer_gateway/_build/test/lib/fluxer_gateway + key: >- + rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock', + 'fluxer_gateway/rebar.config') }} restore-keys: | - rebar3-${{ runner.os }}- + rebar3-${{ runner.os }}-otp28-rebar3.24.0- - name: Check formatting - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step gateway_fmt + run: | + "$FLUXER_CI_BIN" ci --step gateway_fmt - name: Compile - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step gateway_compile + run: | + "$FLUXER_CI_BIN" ci --step gateway_compile - name: Run dialyzer - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step gateway_dialyzer + run: | + "$FLUXER_CI_BIN" ci --step gateway_dialyzer - name: Run eunit tests - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step gateway_eunit + run: | + "$FLUXER_CI_BIN" ci --step gateway_eunit + + - name: Save rebar3 dependencies + if: always() && github.ref == 'refs/heads/main' && steps.rebar3-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + ~/.cache/rebar3 + fluxer_gateway/_build + !fluxer_gateway/_build/default/lib/fluxer_gateway + !fluxer_gateway/_build/test/lib/fluxer_gateway + key: >- + rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock', + 'fluxer_gateway/rebar.config') }} knip: runs-on: ubuntu-24.04 timeout-minutes: 25 + env: + FLUXER_CI_BIN: ${{ github.workspace }}/target/debug/fluxer-ci steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -197,6 +311,19 @@ jobs: toolchain: "1.93.0" targets: wasm32-unknown-unknown + - name: Restore ci helper + id: ci-helper + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: target/debug/fluxer-ci + key: >- + fluxer-ci-bin-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'Cargo.toml', + 'tools/ci/Cargo.toml', 'tools/ci/src/**', 'tools/ci/templates/**') }} + + - name: Build ci helper + if: steps.ci-helper.outputs.cache-hit != 'true' + run: cargo build --locked --package fluxer-ci + - name: Install pnpm uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 @@ -207,14 +334,42 @@ jobs: cache: 'pnpm' - name: Install dependencies - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step install_dependencies + run: | + "$FLUXER_CI_BIN" ci --step install_dependencies + + - name: Restore fluxer_app wasm artifacts + id: app-wasm + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + fluxer_app/pkgs/libfluxcore + fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts + key: >- + app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs', + 'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts', + 'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock', + 'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**', + 'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml', + 'packages/markdown_parser/rust/src/**') }} - name: Run knip - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step knip + run: | + "$FLUXER_CI_BIN" ci --step knip + + - name: Save fluxer_app wasm artifacts + if: always() && github.ref == 'refs/heads/main' && steps.app-wasm.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: | + fluxer_app/pkgs/libfluxcore + fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts + key: >- + app-wasm-${{ runner.os }}-1.93.0-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs', + 'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts', + 'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock', + 'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**', + 'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml', + 'packages/markdown_parser/rust/src/**') }} i18n: runs-on: ubuntu-24.04 @@ -223,12 +378,6 @@ jobs: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - - name: Set up Rust toolchain (CI helpers) - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 - with: - toolchain: "1.93.0" - targets: wasm32-unknown-unknown - - name: Install pnpm uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 @@ -239,9 +388,7 @@ jobs: cache: 'pnpm' - name: Install dependencies - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci - --step install_dependencies + run: pnpm install --frozen-lockfile - name: Compile locale catalogs run: pnpm i18n:compile @@ -276,29 +423,6 @@ jobs: - name: Verify shipped fonts match the lockfile run: python3 tools/fonts/build_fonts.py --verify - ci-scripts: - runs-on: ubuntu-24.04 - timeout-minutes: 25 - steps: - - name: Checkout code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 - with: - toolchain: "1.93.0" - components: rustfmt - - - name: Sync ci helper dependencies - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci-scripts - --step sync - - - name: Run ci helper tests - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- ci-scripts - --step test - helm-and-scripts: runs-on: ubuntu-24.04 timeout-minutes: 10 @@ -306,19 +430,14 @@ jobs: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - - name: Set up Rust toolchain (CI helpers) - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 - with: - toolchain: "1.93.0" - - name: Install helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 - name: Resolve Helm test build version - run: >- - cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- resolve-calver - --github-env - --env-name HELM_TEST_BUILD_VERSION + run: | + set -euo pipefail + read -r YEAR MONTH DAY TIME <<<"$(date -u '+%Y %m %d %H%M%S')" + echo "HELM_TEST_BUILD_VERSION=${YEAR}.$((10#$MONTH))${DAY}.$((10#$TIME))" >> "$GITHUB_ENV" - name: Helm dependency update (all charts) run: | diff --git a/fluxer_api/src/api/auth/services/SudoModeService.ts b/fluxer_api/src/api/auth/services/SudoModeService.ts index b8c56d193..9a36c051c 100644 --- a/fluxer_api/src/api/auth/services/SudoModeService.ts +++ b/fluxer_api/src/api/auth/services/SudoModeService.ts @@ -45,6 +45,10 @@ class SudoModeService { let sudoModeServiceInstance: SudoModeService | null = null; +export function resetSudoModeServiceForTesting(): void { + sudoModeServiceInstance = null; +} + export function getSudoModeService(): SudoModeService { if (!sudoModeServiceInstance) { sudoModeServiceInstance = new SudoModeService(); diff --git a/fluxer_api/src/api/instance/SsoConfigValidation.ts b/fluxer_api/src/api/instance/SsoConfigValidation.ts index 67a6688fa..0cc2ea9b9 100644 --- a/fluxer_api/src/api/instance/SsoConfigValidation.ts +++ b/fluxer_api/src/api/instance/SsoConfigValidation.ts @@ -25,6 +25,10 @@ interface NormalizedSsoConfigValidationResult extends SsoConfigValidationInput { let ssoRequestUrlPolicy: RequestUrlPolicy | null = null; +export function resetSsoRequestUrlPolicyForTesting(): void { + ssoRequestUrlPolicy = null; +} + export function getSsoRequestUrlPolicy(): RequestUrlPolicy { if (ssoRequestUrlPolicy === null) { ssoRequestUrlPolicy = createPublicInternetRequestUrlPolicy({ diff --git a/fluxer_api/src/api/limits/LimitConfigService.ts b/fluxer_api/src/api/limits/LimitConfigService.ts index 0ff8f0815..b4f09e69f 100644 --- a/fluxer_api/src/api/limits/LimitConfigService.ts +++ b/fluxer_api/src/api/limits/LimitConfigService.ts @@ -174,6 +174,10 @@ export class LimitConfigService { } } +export function resetGlobalLimitConfigServiceForTesting(): void { + globalLimitConfigService = null; +} + export function getGlobalLimitConfigSnapshot(): LimitConfigSnapshot { if (!globalLimitConfigService) { throw new Error('LimitConfigService global instance has not been initialized'); diff --git a/fluxer_api/src/api/middleware/BannedAvatarHashCache.ts b/fluxer_api/src/api/middleware/BannedAvatarHashCache.ts index 7ae319cf9..842963b0c 100644 --- a/fluxer_api/src/api/middleware/BannedAvatarHashCache.ts +++ b/fluxer_api/src/api/middleware/BannedAvatarHashCache.ts @@ -89,6 +89,15 @@ class BannedAvatarHashCache { return this.banned.size; } + resetForTesting(): void { + this.shutdown(); + this.banned = new Set(); + this.kvClient = null; + this.subscriberInitialized = false; + this.consecutiveFailures = 0; + this.isInitialized = false; + } + shutdown(): void { if (this.kvSubscription && this.messageHandler) { this.kvSubscription.off('message', this.messageHandler); diff --git a/fluxer_api/src/api/middleware/FileShaCache.ts b/fluxer_api/src/api/middleware/FileShaCache.ts index 26d435eab..53e4439f9 100644 --- a/fluxer_api/src/api/middleware/FileShaCache.ts +++ b/fluxer_api/src/api/middleware/FileShaCache.ts @@ -85,6 +85,15 @@ class FileShaCache { return this.banned.size; } + resetForTesting(): void { + this.shutdown(); + this.banned = new Set(); + this.kvClient = null; + this.subscriberInitialized = false; + this.consecutiveFailures = 0; + this.isInitialized = false; + } + shutdown(): void { if (this.kvSubscription && this.messageHandler) { this.kvSubscription.off('message', this.messageHandler); diff --git a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts b/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts index 0cb7638f9..76c0ab76d 100644 --- a/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts +++ b/fluxer_api/src/api/middleware/PhraseBlocklistCache.ts @@ -144,6 +144,17 @@ export class PhraseBlocklistCache { return this.rawPhraseSet.size; } + resetForTesting(): void { + this.shutdown(); + this.rawPhrases = []; + this.rawPhraseSet = new Set(); + this.rebuildMatchers(); + this.kvClient = null; + this.subscriberInitialized = false; + this.consecutiveFailures = 0; + this.isInitialized = false; + } + shutdown(): void { if (this.kvSubscription && this.messageHandler) { this.kvSubscription.off('message', this.messageHandler); diff --git a/fluxer_api/src/api/middleware/ProfileSubstringBlocklistCache.ts b/fluxer_api/src/api/middleware/ProfileSubstringBlocklistCache.ts index d77132493..f177473dd 100644 --- a/fluxer_api/src/api/middleware/ProfileSubstringBlocklistCache.ts +++ b/fluxer_api/src/api/middleware/ProfileSubstringBlocklistCache.ts @@ -180,6 +180,15 @@ export class ProfileSubstringBlocklistCache { return !!canonical && this.getMatchersForCheck(scope).some((matchers) => matchers.rawSet.has(canonical)); } + resetForTesting(): void { + this.shutdown(); + this.byScope = new Map(); + this.kvClient = null; + this.subscriberInitialized = false; + this.consecutiveFailures = 0; + this.isInitialized = false; + } + shutdown(): void { if (this.kvSubscription && this.messageHandler) { this.kvSubscription.off('message', this.messageHandler); diff --git a/fluxer_api/src/api/middleware/ServiceMiddleware.ts b/fluxer_api/src/api/middleware/ServiceMiddleware.ts index 80b5f0ec4..5fb8945aa 100644 --- a/fluxer_api/src/api/middleware/ServiceMiddleware.ts +++ b/fluxer_api/src/api/middleware/ServiceMiddleware.ts @@ -763,3 +763,15 @@ export const ServiceMiddleware = createMiddleware(async (ctx, next) => ctx.set('ncmecSubmissionService', getNcmecSubmissionService()); await next(); }); + +export function resetServiceMiddlewareForTesting(): void { + shutdownReportService(); + _inboundSmsChallengeService = null; + _registrationEventsRepository = null; + _riskAssessmentRepository = null; + _historicalOutcomeRepository = null; + _suspiciousIpRepository = null; + _ipInfoService = null; + _registrationRiskEvaluator = null; + _liveKitWebhookService = null; +} diff --git a/fluxer_api/src/api/middleware/ServiceRegistry.ts b/fluxer_api/src/api/middleware/ServiceRegistry.ts index 41ea4220d..b80db9161 100644 --- a/fluxer_api/src/api/middleware/ServiceRegistry.ts +++ b/fluxer_api/src/api/middleware/ServiceRegistry.ts @@ -306,3 +306,20 @@ export function getLiveKitServiceInstance(): ILiveKitService | null { export function getVoiceRoomStoreInstance(): IVoiceRoomStore | null { return voiceRoomStoreInstance; } + +export function resetServiceRegistryForTesting(): void { + _kvClient = null; + _snowflakeService = null; + _billingRepository = null; + _blueskyOAuthService = undefined; + _blueskyOAuthInitializationPromise = null; + _blueskyOAuthConfigSignature = null; + _blueskyOAuthInitializationSignature = null; + _disabledBlueskyOAuthService = undefined; + voiceTopology = null; + voiceAvailabilityService = null; + liveKitServiceInstance = null; + voiceRoomStoreInstance = null; + voiceConfigSubscriber = null; + voiceInitializationPromise = null; +} diff --git a/fluxer_api/src/api/middleware/ServiceSingletons.ts b/fluxer_api/src/api/middleware/ServiceSingletons.ts index b7f8cfda4..2613aa09b 100644 --- a/fluxer_api/src/api/middleware/ServiceSingletons.ts +++ b/fluxer_api/src/api/middleware/ServiceSingletons.ts @@ -97,7 +97,7 @@ import { getSnowflakeService, getWorkerService, } from './ServiceRegistry'; -import {singleton} from './Singleton'; +import {clearSingletonsForTesting, singleton} from './Singleton'; export const getUserRepository = singleton(() => new UserRepository(getKVClient())); export const getGuildRepository = singleton(() => new GuildRepository()); @@ -121,7 +121,10 @@ export const getPasswordChangeRepository = singleton(() => new PasswordChangeRep const getUserContactChangeLogRepository = singleton(() => new UserContactChangeLogRepository()); export const getDonationRepository = singleton(() => new DonationRepository()); const getAdminApiKeyRepository = singleton(() => new AdminApiKeyRepository()); -export const getInstanceConfigRepository = singleton(() => new InstanceConfigRepository(getKVClient())); +export const getInstanceConfigRepository = singleton( + () => new InstanceConfigRepository(getKVClient()), + (repository) => repository.shutdown(), +); export const getGatewayRolloutConfigPublisher = singleton( () => new GatewayRolloutConfigPublisher( @@ -200,6 +203,7 @@ const getDownloadsStorageService: () => IStorageService = (() => { export const getErrorI18nService = singleton(() => new ErrorI18nService()); export const getLimitConfigService = singleton( () => new LimitConfigService(getInstanceConfigRepository(), getCacheService(), getKVClient()), + (service) => service.shutdown(), ); export const getPurgeQueue: () => IPurgeQueue = singleton(() => Config.bunny.purgeEnabled ? new BunnyPurgeQueue(getKVClient()) : new NoopPurgeQueue(), @@ -434,3 +438,20 @@ export async function initializeServiceSingletons(): Promise { } await serviceSingletonInitializationPromise; } + +export function resetServiceSingletonsForTesting(): void { + activityTracker?.shutdown(); + clearSingletonsForTesting(); + _virusScanInitPromise = null; + serviceSingletonInitializationPromise = null; + bulkMessageDeletionQueue = null; + bulkMessageDeletionQueueClient = null; + premiumStateQueue = null; + premiumStateQueueClient = null; + activityTracker = null; + activityTrackerClient = null; + activityBuffer = null; + activityBufferClient = null; + accountDeletionQueue = null; + accountDeletionQueueClient = null; +} diff --git a/fluxer_api/src/api/middleware/Singleton.ts b/fluxer_api/src/api/middleware/Singleton.ts index ebf6f5f16..d0d58b081 100644 --- a/fluxer_api/src/api/middleware/Singleton.ts +++ b/fluxer_api/src/api/middleware/Singleton.ts @@ -1,7 +1,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -export function singleton(factory: () => T): () => T { +const singletonClearers: Array<() => void> = []; + +export function singleton(factory: () => T, teardown?: (instance: T) => void): () => T { let instance: T | undefined; + singletonClearers.push(() => { + if (instance !== undefined) { + teardown?.(instance); + } + instance = undefined; + }); return () => { if (instance === undefined) { instance = factory(); @@ -9,3 +17,9 @@ export function singleton(factory: () => T): () => T { return instance; }; } + +export function clearSingletonsForTesting(): void { + for (const clear of singletonClearers) { + clear(); + } +} diff --git a/fluxer_api/src/api/middleware/UrlBlocklistCache.ts b/fluxer_api/src/api/middleware/UrlBlocklistCache.ts index b446d9f5c..95b27aee9 100644 --- a/fluxer_api/src/api/middleware/UrlBlocklistCache.ts +++ b/fluxer_api/src/api/middleware/UrlBlocklistCache.ts @@ -146,6 +146,17 @@ class UrlBlocklistCache { }; } + resetForTesting(): void { + this.shutdown(); + this.exactUrls = new Set(); + this.blockedDomains = new Set(); + this.kvClient = null; + this.storageService = null; + this.subscriberInitialized = false; + this.consecutiveFailures = 0; + this.isInitialized = false; + } + shutdown(): void { if (this.kvSubscription && this.messageHandler) { this.kvSubscription.off('message', this.messageHandler); diff --git a/fluxer_api/src/api/oauth/repositories/ApplicationRepository.ts b/fluxer_api/src/api/oauth/repositories/ApplicationRepository.ts index 216cfdf2a..a65023664 100644 --- a/fluxer_api/src/api/oauth/repositories/ApplicationRepository.ts +++ b/fluxer_api/src/api/oauth/repositories/ApplicationRepository.ts @@ -26,6 +26,10 @@ const FETCH_APPLICATIONS_BY_IDS_CQL = Applications.selectCql({ let cachedAdminSecretHash: string | null = null; +export function resetAdminSecretHashForTesting(): void { + cachedAdminSecretHash = null; +} + async function getAdminSecretHash(): Promise { const secret = Config.admin.oauthClientSecret; if (!secret) { diff --git a/fluxer_api/src/api/risk/IpBanExemptions.ts b/fluxer_api/src/api/risk/IpBanExemptions.ts index f895124b4..ec20dc249 100644 --- a/fluxer_api/src/api/risk/IpBanExemptions.ts +++ b/fluxer_api/src/api/risk/IpBanExemptions.ts @@ -28,3 +28,7 @@ export function isIpBanExempt(ip: string | null | undefined): boolean { const key = getSameIpDecisionKey(ip); return key !== null && getExemptDecisionKeys().has(key); } + +export function resetIpBanExemptionsForTesting(): void { + exemptDecisionKeys = null; +} diff --git a/fluxer_api/src/api/test/ApiTestHarness.ts b/fluxer_api/src/api/test/ApiTestHarness.ts index 453ca4e3c..ce8e145ae 100644 --- a/fluxer_api/src/api/test/ApiTestHarness.ts +++ b/fluxer_api/src/api/test/ApiTestHarness.ts @@ -42,6 +42,7 @@ import {NoopLogger} from './mocks/NoopLogger'; import {NoopUnfurlerService} from './mocks/NoopUnfurlerService'; import {NoopGatewayService} from './NoopGatewayService'; import {NoopWorkerService} from './NoopWorkerService'; +import {resetServiceStateForTesting} from './ResetServiceState'; import {InMemorySearchProvider} from './search/InMemorySearchProvider'; import {TestMediaService} from './TestMediaService'; @@ -91,6 +92,7 @@ export async function createApiTestHarness(options: CreateApiTestHarnessOptions const mockBlueskyOAuthService = new MockBlueskyOAuthService(); setInjectedBlueskyOAuthService(mockBlueskyOAuthService); setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator()); + resetServiceStateForTesting(); const { app, initialize: initializeApp, @@ -152,6 +154,7 @@ export async function createApiTestHarness(options: CreateApiTestHarnessOptions setInjectedMediaService(new TestMediaService(fallbackStorageService)); setInjectedSearchProviderService(new NullSearchProvider()); setInjectedBlueskyOAuthService(new MockBlueskyOAuthService()); + resetServiceStateForTesting(); resetApiServicesForTesting(); } async function requestJson(params: { diff --git a/fluxer_api/src/api/test/ResetServiceState.ts b/fluxer_api/src/api/test/ResetServiceState.ts new file mode 100644 index 000000000..a6a68c302 --- /dev/null +++ b/fluxer_api/src/api/test/ResetServiceState.ts @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {resetSudoModeServiceForTesting} from '../auth/services/SudoModeService'; +import {resetSsoRequestUrlPolicyForTesting} from '../instance/SsoConfigValidation'; +import {resetGlobalLimitConfigServiceForTesting} from '../limits/LimitConfigService'; +import {bannedAvatarHashCache} from '../middleware/BannedAvatarHashCache'; +import {fileShaCache} from '../middleware/FileShaCache'; +import {ipBanCache} from '../middleware/IpBanMiddleware'; +import {phraseBlocklistCache} from '../middleware/PhraseBlocklistCache'; +import {profileSubstringBlocklistCache} from '../middleware/ProfileSubstringBlocklistCache'; +import {resetServiceMiddlewareForTesting} from '../middleware/ServiceMiddleware'; +import {resetServiceRegistryForTesting} from '../middleware/ServiceRegistry'; +import {resetServiceSingletonsForTesting} from '../middleware/ServiceSingletons'; +import {torExitListCache} from '../middleware/TorExitListCache'; +import {urlBlocklistCache} from '../middleware/UrlBlocklistCache'; +import {resetAdminSecretHashForTesting} from '../oauth/repositories/ApplicationRepository'; +import {resetIpBanExemptionsForTesting} from '../risk/IpBanExemptions'; +import {setThemeCssMaxBytesForTesting} from '../theme/ThemeService'; + +export function resetServiceStateForTesting(): void { + resetServiceRegistryForTesting(); + resetServiceSingletonsForTesting(); + resetServiceMiddlewareForTesting(); + resetIpBanExemptionsForTesting(); + resetGlobalLimitConfigServiceForTesting(); + resetSudoModeServiceForTesting(); + resetSsoRequestUrlPolicyForTesting(); + resetAdminSecretHashForTesting(); + setThemeCssMaxBytesForTesting(undefined); + ipBanCache.shutdown(); + ipBanCache.resetCaches(); + torExitListCache.shutdown(); + torExitListCache.clearForTesting(); + urlBlocklistCache.resetForTesting(); + fileShaCache.resetForTesting(); + phraseBlocklistCache.resetForTesting(); + bannedAvatarHashCache.resetForTesting(); + profileSubstringBlocklistCache.resetForTesting(); +} diff --git a/fluxer_api/src/api/theme/ThemeService.ts b/fluxer_api/src/api/theme/ThemeService.ts index a64ca60ab..d141a1799 100644 --- a/fluxer_api/src/api/theme/ThemeService.ts +++ b/fluxer_api/src/api/theme/ThemeService.ts @@ -5,7 +5,20 @@ import {FileSizeTooLargeError} from '@fluxer/errors/src/domains/core/FileSizeToo import {Config} from '../Config'; import type {IStorageService} from '../infrastructure/IStorageService'; -const MAX_CSS_BYTES = 8 * 1024 * 1024; +export const THEME_CSS_MAX_BYTES = 8 * 1024 * 1024; + +let themeCssMaxBytesOverride: number | undefined; + +export function setThemeCssMaxBytesForTesting(bytes: number | undefined): void { + themeCssMaxBytesOverride = bytes; +} + +export function resolveThemeCssMaxBytes(): number { + if (Config.dev.testModeEnabled && themeCssMaxBytesOverride !== undefined) { + return themeCssMaxBytesOverride; + } + return THEME_CSS_MAX_BYTES; +} export class ThemeService { constructor(private readonly storageService: IStorageService) {} @@ -14,7 +27,7 @@ export class ThemeService { id: string; }> { const cssBytes = Buffer.from(css, 'utf-8'); - if (cssBytes.length > MAX_CSS_BYTES) { + if (cssBytes.length > resolveThemeCssMaxBytes()) { throw new FileSizeTooLargeError(); } const themeId = randomBytes(8).toString('hex'); diff --git a/fluxer_api/src/api/theme/tests/ThemeCssSizeLimit.test.ts b/fluxer_api/src/api/theme/tests/ThemeCssSizeLimit.test.ts index 2f3372ccd..6119762ff 100644 --- a/fluxer_api/src/api/theme/tests/ThemeCssSizeLimit.test.ts +++ b/fluxer_api/src/api/theme/tests/ThemeCssSizeLimit.test.ts @@ -2,15 +2,17 @@ import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest'; import {createTestAccount} from '../../auth/tests/AuthTestUtils'; +import {getConfig} from '../../Config'; import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness'; import {HTTP_STATUS} from '../../test/TestConstants'; import {createBuilder} from '../../test/TestRequestBuilder'; +import {resolveThemeCssMaxBytes, setThemeCssMaxBytesForTesting, THEME_CSS_MAX_BYTES} from '../ThemeService'; interface ThemeCreateResponse { id: string; } -const MAX_CSS_BYTES = 8 * 1024 * 1024; +const TEST_MAX_CSS_BYTES = 4096; describe('Theme CSS size limits', () => { let harness: ApiTestHarness; @@ -19,22 +21,33 @@ describe('Theme CSS size limits', () => { }); beforeEach(async () => { await harness.reset(); + setThemeCssMaxBytesForTesting(TEST_MAX_CSS_BYTES); }); afterAll(async () => { + setThemeCssMaxBytesForTesting(undefined); await harness?.shutdown(); }); - it('rejects CSS that exceeds the 8MB limit', async () => { - const user = await createTestAccount(harness); - const oversizedCss = 'a'.repeat(MAX_CSS_BYTES + 1); - await createBuilder(harness, user.token) - .post('/users/@me/themes') - .body({css: oversizedCss}) - .expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE') - .execute(); + it('enforces the 8MB limit in production and only honours the override in test mode', () => { + const config = getConfig(); + expect(THEME_CSS_MAX_BYTES).toBe(8 * 1024 * 1024); + expect(resolveThemeCssMaxBytes()).toBe(TEST_MAX_CSS_BYTES); + config.dev.testModeEnabled = false; + try { + expect(resolveThemeCssMaxBytes()).toBe(THEME_CSS_MAX_BYTES); + } finally { + config.dev.testModeEnabled = true; + } + setThemeCssMaxBytesForTesting(undefined); + try { + expect(resolveThemeCssMaxBytes()).toBe(THEME_CSS_MAX_BYTES); + } finally { + setThemeCssMaxBytesForTesting(TEST_MAX_CSS_BYTES); + } }); - it('accepts CSS at exactly the 8MB limit', async () => { + it('accepts CSS at exactly the real 8MB limit through the full HTTP stack', async () => { + setThemeCssMaxBytesForTesting(undefined); const user = await createTestAccount(harness); - const maxCss = 'a'.repeat(MAX_CSS_BYTES); + const maxCss = 'a'.repeat(THEME_CSS_MAX_BYTES); const theme = await createBuilder(harness, user.token) .post('/users/@me/themes') .body({css: maxCss}) @@ -42,9 +55,38 @@ describe('Theme CSS size limits', () => { .execute(); expect(theme.id).toBeDefined(); }); - it('accepts CSS just under the 8MB limit', async () => { + it('rejects CSS that exceeds the real 8MB limit through the full HTTP stack', async () => { + setThemeCssMaxBytesForTesting(undefined); const user = await createTestAccount(harness); - const nearMaxCss = 'a'.repeat(MAX_CSS_BYTES - 1); + const oversizedCss = 'a'.repeat(THEME_CSS_MAX_BYTES + 1); + await createBuilder(harness, user.token) + .post('/users/@me/themes') + .body({css: oversizedCss}) + .expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE') + .execute(); + }); + it('rejects CSS that exceeds the limit', async () => { + const user = await createTestAccount(harness); + const oversizedCss = 'a'.repeat(TEST_MAX_CSS_BYTES + 1); + await createBuilder(harness, user.token) + .post('/users/@me/themes') + .body({css: oversizedCss}) + .expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE') + .execute(); + }); + it('accepts CSS at exactly the limit', async () => { + const user = await createTestAccount(harness); + const maxCss = 'a'.repeat(TEST_MAX_CSS_BYTES); + const theme = await createBuilder(harness, user.token) + .post('/users/@me/themes') + .body({css: maxCss}) + .expect(HTTP_STATUS.CREATED) + .execute(); + expect(theme.id).toBeDefined(); + }); + it('accepts CSS just under the limit', async () => { + const user = await createTestAccount(harness); + const nearMaxCss = 'a'.repeat(TEST_MAX_CSS_BYTES - 1); const theme = await createBuilder(harness, user.token) .post('/users/@me/themes') .body({css: nearMaxCss}) @@ -52,12 +94,14 @@ describe('Theme CSS size limits', () => { .execute(); expect(theme.id).toBeDefined(); }); - it('rejects CSS exceeding limit with multibyte unicode characters', async () => { + it('rejects CSS whose byte length exceeds the limit even when the character count does not', async () => { const user = await createTestAccount(harness); const unicodeChar = '\u{1F600}'; const bytesPerChar = Buffer.from(unicodeChar, 'utf-8').length; - const charsNeeded = Math.ceil((MAX_CSS_BYTES + 1) / bytesPerChar); + const charsNeeded = Math.ceil((TEST_MAX_CSS_BYTES + 1) / bytesPerChar); const oversizedUnicodeCss = unicodeChar.repeat(charsNeeded); + expect(oversizedUnicodeCss.length).toBeLessThanOrEqual(TEST_MAX_CSS_BYTES); + expect(Buffer.byteLength(oversizedUnicodeCss, 'utf-8')).toBeGreaterThan(TEST_MAX_CSS_BYTES); await createBuilder(harness, user.token) .post('/users/@me/themes') .body({css: oversizedUnicodeCss}) diff --git a/fluxer_api/vitest.config.ts b/fluxer_api/vitest.config.ts index cfb0fc4af..ccfc1d4d5 100644 --- a/fluxer_api/vitest.config.ts +++ b/fluxer_api/vitest.config.ts @@ -16,38 +16,44 @@ function parseParallelInteger(value: string | undefined, fallback: number): numb } function resolveDefaultParallelWorkers(): number { - const halfCoreCountMinusOne = Math.floor(availableParallelism() / 2) - 1; - return Math.max(2, halfCoreCountMinusOne); + return Math.max(2, availableParallelism() - 1); } const DEFAULT_PARALLEL_WORKERS = resolveDefaultParallelWorkers(); const configuredMaxWorkers = parseParallelInteger(process.env.API_TEST_MAX_WORKERS, DEFAULT_PARALLEL_WORKERS); const configuredMaxConcurrency = parseParallelInteger(process.env.API_TEST_MAX_CONCURRENCY, configuredMaxWorkers); +const MODULE_REGISTRY_TEST_FILES = [ + 'src/api/gif/GifRequestCountry.test.ts', + 'src/api/risk/__tests__/AccountPolicyService.test.ts', +]; + +const sharedExclude = [ + ...configDefaults.exclude, + 'pkgs/**', + '../fluxer_desktop/**', + '**/target/**', + '**/*Integration.test.ts', + '**/*ExttestIntegration.test.ts', +]; + +const sharedTestConfig = { + globals: true, + environment: 'node' as const, + setupFiles: ['./src/api/test/Setup.ts'], + pool: 'threads' as const, + testTimeout: 40000, + hookTimeout: 20000, + maxConcurrency: configuredMaxConcurrency, +}; + export default defineConfig({ root: process.cwd(), - plugins: [tsconfigPaths()], cacheDir: './node_modules/.vitest', test: { - globals: true, - environment: 'node', - setupFiles: ['./src/api/test/Setup.ts'], - include: ['src/**/*.{test,spec}.{ts,tsx}'], - exclude: [ - ...configDefaults.exclude, - 'pkgs/**', - '../fluxer_desktop/**', - '**/target/**', - '**/*Integration.test.ts', - '**/*ExttestIntegration.test.ts', - ], - pool: 'threads', - fileParallelism: true, maxWorkers: configuredMaxWorkers, maxConcurrency: configuredMaxConcurrency, - isolate: true, - testTimeout: 40000, - hookTimeout: 20000, + fileParallelism: true, reporters: ['default', 'json'], outputFile: './test-results.json', coverage: { @@ -63,5 +69,27 @@ export default defineConfig({ '**/TestHelpers.tsx', ], }, + projects: [ + { + plugins: [tsconfigPaths()], + test: { + ...sharedTestConfig, + name: 'api', + include: ['src/**/*.{test,spec}.{ts,tsx}'], + exclude: [...sharedExclude, ...MODULE_REGISTRY_TEST_FILES], + isolate: false, + }, + }, + { + plugins: [tsconfigPaths()], + test: { + ...sharedTestConfig, + name: 'api-module-registry', + include: MODULE_REGISTRY_TEST_FILES, + exclude: sharedExclude, + isolate: true, + }, + }, + ], }, }); diff --git a/fluxer_app/package.json b/fluxer_app/package.json index 108dfe1b2..5855ceba9 100644 --- a/fluxer_app/package.json +++ b/fluxer_app/package.json @@ -16,7 +16,7 @@ }, "scripts": { "build": "pnpm wasm:codegen && pnpm generate:colors && pnpm generate:message-layout && pnpm generate:theme-variables && pnpm generate:masks && pnpm generate:css-types && pnpm lingui:compile && rm -rf dist && rspack build --mode production && pnpm tsx scripts/build-sw.mjs", - "dev": "TOKIO_WORKER_THREADS=4 RAYON_NUM_THREADS=4 cargo run --manifest-path ../tools/ci/Cargo.toml -- app-dev-server", + "dev": "TOKIO_WORKER_THREADS=4 RAYON_NUM_THREADS=4 ../tools/ci/run.sh app-dev-server", "test": "vitest run", "bench:member-list": "vitest bench --run src/features/member/utils/MemberListRangeUtils.bench.ts", "bench:gif-picker": "vitest bench --run src/features/channel/components/pickers/gif/GifPickerStateMachine.bench.ts src/features/channel/components/pickers/gif/GifPickerGridData.bench.ts src/features/channel/components/pickers/gif/GifPickerLoadingSkeletonGridLayout.bench.ts src/features/channel/components/pickers/shared/TileFlowSolver.bench.ts", @@ -46,7 +46,7 @@ "wasm:bench:download-realmedia": "node scripts/BenchLibfluxcore.mjs --download-realmedia", "wasm:bench:highres": "pnpm wasm:codegen && node scripts/BenchLibfluxcore.mjs --profile=highres", "wasm:bench:realmedia": "pnpm wasm:codegen && node scripts/BenchLibfluxcore.mjs --profile=realmedia --offline", - "wasm:codegen": "cargo run --locked --manifest-path ../tools/ci/Cargo.toml -- build-app-wasm", + "wasm:codegen": "../tools/ci/run.sh build-app-wasm", "wasm:codegen:simd": "FLUXCORE_WASM_SIMD=1 pnpm wasm:codegen" }, "browserslist": [ diff --git a/fluxer_desktop/package.json b/fluxer_desktop/package.json index d1e3dc191..1663721a2 100644 --- a/fluxer_desktop/package.json +++ b/fluxer_desktop/package.json @@ -10,7 +10,7 @@ "main": "dist/main/index.js", "scripts": { "build": "node scripts/build.mjs", - "set-channel": "cargo run --manifest-path ../tools/ci/Cargo.toml -- build-desktop --step set_build_channel", + "set-channel": "../tools/ci/run.sh build-desktop --step set_build_channel", "test": "cargo test --manifest-path native/rust/Cargo.toml", "typecheck": "tsgo --noEmit" }, diff --git a/fluxer_gateway/rebar.config b/fluxer_gateway/rebar.config index 8ae82e969..99a565e7b 100644 --- a/fluxer_gateway/rebar.config +++ b/fluxer_gateway/rebar.config @@ -17,10 +17,7 @@ {rebar3_lint, {git, "https://github.com/project-fifo/rebar3_lint.git", {branch, "master"}}} ]}. -{pre_hooks, [ - {compile, - "cargo run --locked --quiet --manifest-path ../tools/ci/Cargo.toml -- build-gateway-nifs --gateway-dir ."} -]}. +{pre_hooks, [{compile, "../tools/ci/run.sh build-gateway-nifs --gateway-dir ."}]}. {overrides, [ {override, cowboy, [{deps, [cowlib, ranch]}]}, diff --git a/packages/schema/package.json b/packages/schema/package.json index fd104c9ec..d813d6155 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -11,7 +11,7 @@ "test:coverage": "vitest run --coverage", "test:watch": "vitest", "typecheck": "tsgo --noEmit", - "generate": "cd proto && buf generate && cargo run --manifest-path ../../../tools/ci/Cargo.toml -- clean-schema-generated-files --root ../src/gen && cd .. && pnpm exec biome check --write --unsafe --no-errors-on-unmatched src/gen" + "generate": "cd proto && buf generate && ../../../tools/ci/run.sh clean-schema-generated-files --root ../src/gen && cd .. && pnpm exec biome check --write --unsafe --no-errors-on-unmatched src/gen" }, "dependencies": { "@bufbuild/protobuf": "^2.12.0", diff --git a/tools/ci/run.sh b/tools/ci/run.sh new file mode 100755 index 000000000..614d0c9bc --- /dev/null +++ b/tools/ci/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -euo pipefail + +if [ -x "${FLUXER_CI_BIN:-}" ]; then + exec "$FLUXER_CI_BIN" "$@" +fi + +script_dir="$(cd "$(dirname "$0")" && pwd)" +exec cargo run --quiet --manifest-path "$script_dir/Cargo.toml" -- "$@" diff --git a/tools/ci/src/app_proxy.rs b/tools/ci/src/app_proxy.rs index a1c5baac3..6f533b67c 100644 --- a/tools/ci/src/app_proxy.rs +++ b/tools/ci/src/app_proxy.rs @@ -147,13 +147,16 @@ fn build_and_extract_command() -> Result { ) .env( "CACHE_FROM", - env::var("CACHE_FROM") - .unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy".to_string()), + env::var("CACHE_FROM").unwrap_or_else(|_| default_app_proxy_cache_ref()), ) .env( "CACHE_TO", - env::var("CACHE_TO") - .unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy,mode=max".to_string()), + env::var("CACHE_TO").unwrap_or_else(|_| { + format!( + "{},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true", + default_app_proxy_cache_ref() + ) + }), ) .env( "DOCKER_BUILD_SUMMARY", @@ -165,6 +168,11 @@ fn build_and_extract_command() -> Result { )) } +fn default_app_proxy_cache_ref() -> String { + let owner = ghcr_owner().unwrap_or_else(|_| "fluxerapp".to_string()); + format!("type=registry,ref=ghcr.io/{owner}/fluxer-app-proxy:buildcache-amd64") +} + fn ghcr_owner() -> Result { for key in ["GHCR_OWNER", "GITHUB_REPOSITORY_OWNER", "OWNER"] { if let Ok(value) = env::var(key) { diff --git a/tools/ci/src/app_wasm.rs b/tools/ci/src/app_wasm.rs index 91ed4bdfd..a65075897 100644 --- a/tools/ci/src/app_wasm.rs +++ b/tools/ci/src/app_wasm.rs @@ -142,8 +142,7 @@ fn build_markdown_parser_wasm(app_dir: &Path) -> Result<()> { let rust_source_dir = app_dir.join("../packages/markdown_parser/rust"); let bytes_path = app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts"); - let temp = TempDir::new().context("Failed to create source temp directory")?; - let target_dir = temp.path().join("target"); + let target_dir = rust_source_dir.join("target"); run_command(apply_wasm_c_toolchain( CommandSpec::new("cargo") diff --git a/tools/ci/src/ci_workflow.rs b/tools/ci/src/ci_workflow.rs index d6051bae1..ef5ab79b3 100644 --- a/tools/ci/src/ci_workflow.rs +++ b/tools/ci/src/ci_workflow.rs @@ -6,6 +6,7 @@ use crate::gateway::{GatewayStep, run_gateway_step}; use anyhow::{Context, Result}; use clap::{Args, ValueEnum}; use std::env; +use std::ffi::OsString; use std::path::{Path, PathBuf}; #[derive(Debug, Args, Clone)] @@ -27,17 +28,10 @@ enum CiStep { GatewayEunit, } -#[derive(Debug, Args, Clone)] -pub struct CiScriptsArgs { - #[arg(long, value_enum)] - step: CiScriptsStep, -} - -#[derive(Debug, Clone, Copy, ValueEnum)] -#[clap(rename_all = "snake_case")] -enum CiScriptsStep { - Sync, - Test, +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AppWasm { + Build, + ReuseIfPresent, } pub async fn run_ci(args: CiArgs) -> Result<()> { @@ -51,7 +45,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> { CiStep::Typecheck => { ensure_desktop_build_channel_file(&root)?; run_generators(&root, true)?; - run_app_test_artifact_generators(&root)?; + run_app_test_artifact_generators(&root, AppWasm::Build)?; run_command( CommandSpec::new("pnpm") .args(["-r", "--if-present", "typecheck"]) @@ -60,7 +54,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> { } CiStep::Test => { run_generators(&root, false)?; - run_app_test_artifact_generators(&root)?; + run_app_test_artifact_generators(&root, AppWasm::ReuseIfPresent)?; run_workspace_tests(&root)?; run_command(with_test_env( CommandSpec::new("pnpm") @@ -69,7 +63,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> { )) } CiStep::Knip => { - run_app_test_artifact_generators(&root)?; + run_app_test_artifact_generators(&root, AppWasm::ReuseIfPresent)?; ensure_desktop_build_channel_file(&root)?; run_fluxer_app_script(&root, "i18n:compile")?; run_command( @@ -98,11 +92,43 @@ fn ensure_desktop_build_channel_file(root: &Path) -> Result<()> { write_build_channel_file(&root.join("fluxer_desktop"), &channel) } -fn run_app_test_artifact_generators(root: &Path) -> Result<()> { - run_fluxer_app_script(root, "wasm:codegen")?; +fn run_app_test_artifact_generators(root: &Path, wasm: AppWasm) -> Result<()> { + match (wasm, missing_app_wasm_artifact(root)) { + (AppWasm::ReuseIfPresent, None) => { + println!("Reusing restored fluxer_app wasm artifacts"); + } + (AppWasm::ReuseIfPresent, Some(missing)) => { + println!( + "Rebuilding fluxer_app wasm artifacts: {} is missing", + missing.display() + ); + run_fluxer_app_script(root, "wasm:codegen")?; + } + (AppWasm::Build, _) => run_fluxer_app_script(root, "wasm:codegen")?, + } run_fluxer_app_script(root, "generate:masks") } +fn app_wasm_artifacts(root: &Path) -> Vec { + let app_dir = root.join("fluxer_app"); + vec![ + app_dir.join("pkgs/libfluxcore/libfluxcore.js"), + app_dir.join("pkgs/libfluxcore/libfluxcore.d.ts"), + app_dir.join("pkgs/libfluxcore/libfluxcore_bindgen.js"), + app_dir.join("pkgs/libfluxcore/libfluxcore_bindgen.d.ts"), + app_dir.join("pkgs/libfluxcore/libfluxcore_bg.wasm"), + app_dir.join("pkgs/libfluxcore/libfluxcore_bg.wasm.d.ts"), + app_dir.join("pkgs/libfluxcore/package.json"), + app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts"), + ] +} + +fn missing_app_wasm_artifact(root: &Path) -> Option { + app_wasm_artifacts(root) + .into_iter() + .find(|path| !path.exists()) +} + fn run_fluxer_app_script(root: &Path, script: &str) -> Result<()> { run_command( CommandSpec::new("pnpm") @@ -111,27 +137,6 @@ fn run_fluxer_app_script(root: &Path, script: &str) -> Result<()> { ) } -pub async fn run_ci_scripts(args: CiScriptsArgs) -> Result<()> { - let root = repo_root()?; - match args.step { - CiScriptsStep::Sync => run_command( - CommandSpec::new("cargo") - .args([ - "fetch", - "--locked", - "--manifest-path", - "tools/ci/Cargo.toml", - ]) - .current_dir(root), - ), - CiScriptsStep::Test => run_command( - CommandSpec::new("cargo") - .args(["test", "--locked", "--manifest-path", "tools/ci/Cargo.toml"]) - .current_dir(root), - ), - } -} - fn run_generators(root: &Path, for_typecheck: bool) -> Result<()> { for command in generator_commands(for_typecheck) { run_command(command.current_dir(root))?; @@ -155,28 +160,41 @@ fn generator_commands(for_typecheck: bool) -> Vec { commands } +fn workspace_test_args(concurrency: Option<&str>) -> Vec { + let mut args = vec![OsString::from("-r")]; + if let Some(concurrency) = concurrency { + args.push(OsString::from(format!( + "--workspace-concurrency={concurrency}" + ))); + } + args.extend( + [ + "--filter", + "!fluxer_api", + "--filter", + "!fluxer", + "--filter", + "!fluxer_desktop", + "--if-present", + "test", + ] + .into_iter() + .map(OsString::from), + ); + args +} + fn run_workspace_tests(root: &Path) -> Result<()> { - let workspace_concurrency = - env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").unwrap_or_else(|_| "2".to_string()); + let concurrency = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").ok(); run_command(with_test_env( CommandSpec::new("pnpm") - .args([ - "-r", - &format!("--workspace-concurrency={workspace_concurrency}"), - "--filter", - "!fluxer_api", - "--filter", - "!fluxer", - "--if-present", - "test", - ]) + .args(workspace_test_args(concurrency.as_deref())) .current_dir(root), )) } fn with_test_env(spec: CommandSpec) -> CommandSpec { let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url()); - let api_workers = env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| "2".to_string()); spec.env("FLUXER_NATS_URL", &nats_url) .env( "FLUXER_NATS_CORE_URL", @@ -186,11 +204,6 @@ fn with_test_env(spec: CommandSpec) -> CommandSpec { "FLUXER_NATS_JETSTREAM_URL", env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone()), ) - .env("API_TEST_MAX_WORKERS", &api_workers) - .env( - "API_TEST_MAX_CONCURRENCY", - env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers), - ) } fn default_test_nats_url() -> String { @@ -211,7 +224,6 @@ fn repo_root() -> Result { #[cfg(test)] mod tests { use super::*; - use std::ffi::OsString; #[test] fn generator_commands_include_i18n_types_only_for_typecheck() { @@ -237,7 +249,7 @@ mod tests { } #[test] - fn with_test_env_sets_all_nats_urls_and_concurrency() { + fn with_test_env_sets_all_nats_urls_and_leaves_worker_counts_to_vitest() { let spec = with_test_env(CommandSpec::new("pnpm")); let env = spec .env @@ -257,13 +269,49 @@ mod tests { env.get(&OsString::from("FLUXER_NATS_JETSTREAM_URL")), Some(&default_nats_url) ); - assert_eq!( - env.get(&OsString::from("API_TEST_MAX_WORKERS")), - Some(&OsString::from("2")) + assert!(!env.contains_key(&OsString::from("API_TEST_MAX_WORKERS"))); + assert!(!env.contains_key(&OsString::from("API_TEST_MAX_CONCURRENCY"))); + } + + #[test] + fn workspace_tests_exclude_desktop_and_leave_concurrency_to_pnpm() { + let args = workspace_test_args(None); + + assert_eq!(args[0], OsString::from("-r")); + assert!( + !args + .iter() + .any(|arg| arg.to_string_lossy().starts_with("--workspace-concurrency")) ); + assert!(args.windows(2).any(|pair| pair + == [ + OsString::from("--filter"), + OsString::from("!fluxer_desktop") + ])); + } + + #[test] + fn workspace_tests_forward_an_explicit_concurrency_override() { + let args = workspace_test_args(Some("4")); + + assert_eq!(args[1], OsString::from("--workspace-concurrency=4")); + } + + #[test] + fn missing_app_wasm_artifact_reports_the_first_absent_output() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + assert_eq!( - env.get(&OsString::from("API_TEST_MAX_CONCURRENCY")), - Some(&OsString::from("2")) + missing_app_wasm_artifact(root), + Some(root.join("fluxer_app/pkgs/libfluxcore/libfluxcore.js")) ); + + for path in app_wasm_artifacts(root) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, "").unwrap(); + } + + assert_eq!(missing_app_wasm_artifact(root), None); } } diff --git a/tools/ci/src/desktop.rs b/tools/ci/src/desktop.rs index ccfc890fb..9765ba3f1 100644 --- a/tools/ci/src/desktop.rs +++ b/tools/ci/src/desktop.rs @@ -122,14 +122,14 @@ const PLATFORMS: &[Platform] = &[ platform: "windows", arch: "x64", desktop_variant: DEFAULT_DESKTOP_VARIANT, - os: "blacksmith-32vcpu-windows-2025", + os: "windows-2025", electron_arch: "x64", }, Platform { platform: "windows", arch: "arm64", desktop_variant: DEFAULT_DESKTOP_VARIANT, - os: "blacksmith-32vcpu-windows-2025", + os: "windows-2025", electron_arch: "arm64", }, Platform { @@ -143,14 +143,14 @@ const PLATFORMS: &[Platform] = &[ platform: "linux", arch: "x64", desktop_variant: DEFAULT_DESKTOP_VARIANT, - os: "blacksmith-32vcpu-ubuntu-2404", + os: "ubuntu-24.04", electron_arch: "x64", }, Platform { platform: "linux", arch: "arm64", desktop_variant: DEFAULT_DESKTOP_VARIANT, - os: "blacksmith-32vcpu-ubuntu-2404-arm", + os: "ubuntu-24.04-arm", electron_arch: "arm64", }, ]; @@ -3520,9 +3520,9 @@ mod tests { assert_eq!( selected, vec![ - "{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-windows-2025\",\"electron_arch\":\"arm64\"}", - "{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404\",\"electron_arch\":\"x64\"}", - "{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404-arm\",\"electron_arch\":\"arm64\"}", + "{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"windows-2025\",\"electron_arch\":\"arm64\"}", + "{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04\",\"electron_arch\":\"x64\"}", + "{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04-arm\",\"electron_arch\":\"arm64\"}", ] ); } @@ -3560,9 +3560,9 @@ mod tests { assert_eq!( selected, vec![ - "{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-windows-2025\",\"electron_arch\":\"arm64\"}", - "{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404\",\"electron_arch\":\"x64\"}", - "{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404-arm\",\"electron_arch\":\"arm64\"}", + "{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"windows-2025\",\"electron_arch\":\"arm64\"}", + "{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04\",\"electron_arch\":\"x64\"}", + "{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04-arm\",\"electron_arch\":\"arm64\"}", ] ); } diff --git a/tools/ci/src/lib.rs b/tools/ci/src/lib.rs index 47eeb0f52..41c79f502 100644 --- a/tools/ci/src/lib.rs +++ b/tools/ci/src/lib.rs @@ -34,7 +34,6 @@ enum Command { BuildDesktopNativeAddon(desktop_native::BuildDesktopNativeAddonArgs), BuildGatewayNifs(gateway::BuildGatewayNifsArgs), Ci(ci_workflow::CiArgs), - CiScripts(ci_workflow::CiScriptsArgs), CleanSchemaGeneratedFiles(schema::CleanSchemaGeneratedFilesArgs), Gateway(gateway::GatewayArgs), Release(release::ReleaseArgs), @@ -53,7 +52,6 @@ pub async fn run() -> Result<()> { } Command::BuildGatewayNifs(args) => gateway::run_build_gateway_nifs(args), Command::Ci(args) => ci_workflow::run_ci(args).await, - Command::CiScripts(args) => ci_workflow::run_ci_scripts(args).await, Command::CleanSchemaGeneratedFiles(args) => schema::run_clean_generated_files(args), Command::Gateway(args) => gateway::run_gateway(args), Command::Release(args) => release::run(args).await, diff --git a/tools/dev/src/tasks.rs b/tools/dev/src/tasks.rs index f935f305a..45d70bf20 100644 --- a/tools/dev/src/tasks.rs +++ b/tools/dev/src/tasks.rs @@ -6,9 +6,6 @@ use anyhow::Result; use std::env; use std::path::Path; -const DEFAULT_TEST_WORKSPACE_CONCURRENCY: &str = "2"; -const DEFAULT_API_TEST_WORKERS: &str = "2"; - fn task_run(args: &[&str]) -> Result<()> { run_command( args, @@ -22,8 +19,6 @@ fn task_run(args: &[&str]) -> Result<()> { fn test_env() -> Vec<(String, Option)> { let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url()); - let api_workers = - env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| DEFAULT_API_TEST_WORKERS.to_owned()); vec![ ("FLUXER_NATS_URL".to_owned(), Some(nats_url.clone())), ( @@ -32,12 +27,7 @@ fn test_env() -> Vec<(String, Option)> { ), ( "FLUXER_NATS_JETSTREAM_URL".to_owned(), - Some(env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone())), - ), - ("API_TEST_MAX_WORKERS".to_owned(), Some(api_workers.clone())), - ( - "API_TEST_MAX_CONCURRENCY".to_owned(), - Some(env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers)), + Some(env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or(nats_url)), ), ] } @@ -68,27 +58,39 @@ pub fn run_typecheck() -> Result { pub fn run_test() -> Result { run_generators(false)?; - let workspace_concurrency = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY") - .unwrap_or_else(|_| DEFAULT_TEST_WORKSPACE_CONCURRENCY.to_owned()); - let env = test_env(); - run_command( - &[ - "pnpm", - "-r", - &format!("--workspace-concurrency={workspace_concurrency}"), + let mut args = vec!["pnpm".to_owned(), "-r".to_owned()]; + if let Ok(concurrency) = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY") { + args.push(format!("--workspace-concurrency={concurrency}")); + } + args.extend( + [ "--filter", "!fluxer_api", "--filter", "!fluxer", + "--filter", + "!fluxer_desktop", "--if-present", "test", - ], + ] + .into_iter() + .map(str::to_owned), + ); + let env = test_env(); + run_command( + &args.iter().map(String::as_str).collect::>(), RunOptions { env: env.clone(), load_default_env: false, ..RunOptions::default() }, )?; + task_run(&[ + "cargo", + "test", + "--manifest-path", + "fluxer_desktop/native/rust/Cargo.toml", + ])?; run_command( &["pnpm", "--filter", "fluxer_api", "test"], RunOptions {