mirror of
https://github.com/fluxerapp/fluxer.git
synced 2026-09-02 21:04:06 +03:00
workspace: reconcile latest Fluxer into the open source release and continue development there
This commit is contained in:
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Parse criterion output and compare against a baseline.json.
|
||||
|
||||
Reads:
|
||||
argv[1] path to the captured `cargo bench` log
|
||||
BASELINE_PATH (env) path to baseline.json
|
||||
BUDGET_PERCENT (env) allowed slowdown in percent (e.g. 5 means +5%)
|
||||
|
||||
Exits 0 if every bench mean is within +BUDGET of baseline median; 1 otherwise.
|
||||
Unknown benches in the log are reported but do not fail the run.
|
||||
Benches in the baseline that did not appear in the log are reported and fail.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
UNIT_NS = {
|
||||
"ns": 1.0,
|
||||
"us": 1_000.0,
|
||||
"µs": 1_000.0,
|
||||
"ms": 1_000_000.0,
|
||||
"s": 1_000_000_000.0,
|
||||
}
|
||||
|
||||
LINE_WITH_NAME_RE = re.compile(
|
||||
r"^(?P<name>\S[^\t]*?)\s+time:\s*"
|
||||
r"\[(?P<low>[\d.]+)\s*(?P<low_u>\S+)\s+"
|
||||
r"(?P<mean>[\d.]+)\s*(?P<mean_u>\S+)\s+"
|
||||
r"(?P<high>[\d.]+)\s*(?P<high_u>\S+)\]"
|
||||
)
|
||||
|
||||
TIME_ONLY_RE = re.compile(
|
||||
r"^\s*time:\s*"
|
||||
r"\[(?P<low>[\d.]+)\s*(?P<low_u>\S+)\s+"
|
||||
r"(?P<mean>[\d.]+)\s*(?P<mean_u>\S+)\s+"
|
||||
r"(?P<high>[\d.]+)\s*(?P<high_u>\S+)\]"
|
||||
)
|
||||
|
||||
|
||||
def to_ns(value: float, unit: str) -> float:
|
||||
if unit not in UNIT_NS:
|
||||
raise ValueError(f"unknown criterion time unit: {unit!r}")
|
||||
return value * UNIT_NS[unit]
|
||||
|
||||
|
||||
def parse_log(log_path: Path) -> dict[str, dict[str, float]]:
|
||||
measurements: dict[str, dict[str, float]] = {}
|
||||
name_buffer: str | None = None
|
||||
for raw in log_path.read_text(errors="replace").splitlines():
|
||||
line = raw.rstrip()
|
||||
if not line:
|
||||
continue
|
||||
if "change:" in line:
|
||||
continue
|
||||
if "time:" in line:
|
||||
match = LINE_WITH_NAME_RE.match(line)
|
||||
if match:
|
||||
measurements[match.group("name").strip()] = build_measurement(match)
|
||||
name_buffer = None
|
||||
continue
|
||||
match = TIME_ONLY_RE.match(line)
|
||||
if match and name_buffer is not None:
|
||||
measurements[name_buffer] = build_measurement(match)
|
||||
name_buffer = None
|
||||
continue
|
||||
if line.startswith("Benchmarking "):
|
||||
name = line[len("Benchmarking "):].strip()
|
||||
if name.endswith(": Analyzing"):
|
||||
name_buffer = name[: -len(": Analyzing")]
|
||||
elif ":" not in name:
|
||||
name_buffer = name
|
||||
continue
|
||||
return measurements
|
||||
|
||||
|
||||
def build_measurement(match: re.Match[str]) -> dict[str, float]:
|
||||
return {
|
||||
"low_ns": to_ns(float(match.group("low")), match.group("low_u")),
|
||||
"mean_ns": to_ns(float(match.group("mean")), match.group("mean_u")),
|
||||
"high_ns": to_ns(float(match.group("high")), match.group("high_u")),
|
||||
}
|
||||
|
||||
|
||||
def compare(
|
||||
baseline: dict,
|
||||
measurements: dict[str, dict[str, float]],
|
||||
budget_percent: float,
|
||||
) -> tuple[list[str], list[str], list[str]]:
|
||||
benches = baseline.get("benches", {})
|
||||
if not isinstance(benches, dict):
|
||||
raise ValueError("baseline.json missing 'benches' object")
|
||||
regressions: list[str] = []
|
||||
passes: list[str] = []
|
||||
missing: list[str] = []
|
||||
for name, entry in benches.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if "median_ns" not in entry:
|
||||
continue
|
||||
baseline_ns = float(entry["median_ns"])
|
||||
measured = measurements.get(name)
|
||||
if measured is None:
|
||||
missing.append(name)
|
||||
continue
|
||||
observed_ns = measured["mean_ns"]
|
||||
delta = observed_ns - baseline_ns
|
||||
delta_pct = (delta / baseline_ns) * 100.0 if baseline_ns > 0 else 0.0
|
||||
effective_budget = float(entry.get("budget_percent_override", budget_percent))
|
||||
budget_ns = baseline_ns * (effective_budget / 100.0)
|
||||
status = "OK" if delta <= budget_ns else "REGRESSION"
|
||||
summary = (
|
||||
f" {status:11s} {name}: baseline={baseline_ns:.3f}ns "
|
||||
f"observed={observed_ns:.3f}ns delta={delta_pct:+.2f}% "
|
||||
f"(budget=+{effective_budget:.1f}%)"
|
||||
)
|
||||
if status == "OK":
|
||||
passes.append(summary)
|
||||
else:
|
||||
regressions.append(summary)
|
||||
return passes, regressions, missing
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: _compare_criterion.py <bench-log>", file=sys.stderr)
|
||||
return 2
|
||||
log_path = Path(sys.argv[1])
|
||||
baseline_path = Path(os.environ["BASELINE_PATH"])
|
||||
budget = float(os.environ.get("BUDGET_PERCENT", "5"))
|
||||
strict_missing = os.environ.get("STRICT_MISSING", "0") == "1"
|
||||
baseline = json.loads(baseline_path.read_text())
|
||||
measurements = parse_log(log_path)
|
||||
if not measurements:
|
||||
print("ERROR: no criterion measurements parsed from log", file=sys.stderr)
|
||||
return 2
|
||||
passes, regressions, missing = compare(baseline, measurements, budget)
|
||||
for line in passes:
|
||||
print(line)
|
||||
for line in regressions:
|
||||
print(line)
|
||||
if missing and strict_missing:
|
||||
print("MISSING in run (baseline expected but no measurement found):")
|
||||
for name in missing:
|
||||
print(f" {name}")
|
||||
elif missing:
|
||||
print(
|
||||
f"(info: {len(missing)} baseline entries not seen in this run; "
|
||||
"run other bench files in the same crate to cover them)"
|
||||
)
|
||||
if regressions or (missing and strict_missing):
|
||||
print(
|
||||
f"FAIL: {len(regressions)} regression(s), "
|
||||
f"{len(missing) if strict_missing else 0} missing bench(es)"
|
||||
)
|
||||
return 1
|
||||
print(
|
||||
f"OK: {len(passes)} bench(es) within budget "
|
||||
f"(default +{budget:.1f}%, per-bench overrides where noted)"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
DEFAULT_CRATES=$(cat <<'ENTRIES'
|
||||
fluxer_desktop/native/rt-thread:tick
|
||||
fluxer_desktop/native/audio-mix:mix
|
||||
fluxer_desktop/native/screen-frame-bus:staging
|
||||
fluxer_desktop/native/screen-frame-bus:frame_pool
|
||||
fluxer_desktop/native/nv12-gpu-pack:pack
|
||||
fluxer_desktop/native/encoder-ring:ring
|
||||
fluxer_desktop/native/linux-audio-capture:end_to_end
|
||||
fluxer_desktop/native/linux-screen-capture:pipewire_callback
|
||||
fluxer_desktop/native/webrtc-sender:frame_bus
|
||||
fluxer_desktop/native/rust:native_core
|
||||
ENTRIES
|
||||
)
|
||||
|
||||
ENTRIES="${BENCH_CRATES:-$DEFAULT_CRATES}"
|
||||
|
||||
log() { printf '[check-all] %s\n' "$*" >&2; }
|
||||
|
||||
pass_count=0
|
||||
fail_count=0
|
||||
skip_count=0
|
||||
failed_names=()
|
||||
|
||||
while IFS= read -r entry; do
|
||||
[ -n "$entry" ] || continue
|
||||
case "$entry" in
|
||||
\#*) continue;;
|
||||
esac
|
||||
crate="${entry%%:*}"
|
||||
bench="${entry##*:}"
|
||||
if [ "${BENCH_SKIP_GPU:-0}" = "1" ] && [ "$crate" = "fluxer_desktop/native/nv12-gpu-pack" ]; then
|
||||
log "skip (gpu): $crate $bench"
|
||||
skip_count=$((skip_count + 1))
|
||||
continue
|
||||
fi
|
||||
log "==> $crate :: $bench"
|
||||
if "$SCRIPT_DIR/check-regression.sh" "$REPO_ROOT/$crate" "$bench"; then
|
||||
pass_count=$((pass_count + 1))
|
||||
else
|
||||
fail_count=$((fail_count + 1))
|
||||
failed_names+=("$crate::$bench")
|
||||
fi
|
||||
done <<< "$ENTRIES"
|
||||
|
||||
printf '\n[check-all] summary: %d passed, %d failed, %d skipped\n' \
|
||||
"$pass_count" "$fail_count" "$skip_count"
|
||||
|
||||
if [ "$fail_count" -gt 0 ]; then
|
||||
printf '[check-all] failed entries:\n'
|
||||
for name in "${failed_names[@]}"; do
|
||||
printf ' - %s\n' "$name"
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BUDGET="${BENCH_REGRESSION_BUDGET_PERCENT:-5}"
|
||||
MEASUREMENT_TIME="${BENCH_MEASUREMENT_TIME:-5}"
|
||||
WARM_UP_TIME="${BENCH_WARM_UP_TIME:-2}"
|
||||
|
||||
log() { printf '[check-regression] %s\n' "$*" >&2; }
|
||||
fail() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
fail "usage: $0 <crate-path> <bench-name>"
|
||||
fi
|
||||
|
||||
CRATE_PATH="$1"
|
||||
BENCH_NAME="$2"
|
||||
|
||||
[ -d "$CRATE_PATH" ] || fail "crate path not a directory: $CRATE_PATH"
|
||||
[ -f "$CRATE_PATH/Cargo.toml" ] || fail "no Cargo.toml in $CRATE_PATH"
|
||||
BASELINE="$CRATE_PATH/benches/baseline.json"
|
||||
[ -f "$BASELINE" ] || fail "no baseline.json at $BASELINE"
|
||||
command -v cargo >/dev/null || fail "cargo not on PATH"
|
||||
command -v python3 >/dev/null || fail "python3 not on PATH"
|
||||
|
||||
LOG_FILE="$(mktemp -t bench-regression.XXXXXX.log)"
|
||||
trap 'rm -f "$LOG_FILE"' EXIT
|
||||
|
||||
log "running cargo bench --bench $BENCH_NAME in $CRATE_PATH"
|
||||
(
|
||||
cd "$CRATE_PATH"
|
||||
cargo bench --bench "$BENCH_NAME" -- \
|
||||
--warm-up-time "$WARM_UP_TIME" \
|
||||
--measurement-time "$MEASUREMENT_TIME"
|
||||
) >"$LOG_FILE" 2>&1 || {
|
||||
tail -50 "$LOG_FILE" >&2
|
||||
fail "cargo bench failed; see log above"
|
||||
}
|
||||
|
||||
BUDGET_PERCENT="$BUDGET" BASELINE_PATH="$BASELINE" \
|
||||
python3 "$(dirname "$0")/_compare_criterion.py" "$LOG_FILE"
|
||||
Generated
+3436
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "fluxer-ci"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0-or-later"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.102"
|
||||
aws-config = "1.8.18"
|
||||
aws-sdk-s3 = "1.135.0"
|
||||
base64 = "0.22.1"
|
||||
bytes = "1.11.1"
|
||||
chrono = "0.4.45"
|
||||
clap = { version = "4.6.1", features = ["derive"] }
|
||||
flate2 = "1.1.9"
|
||||
hex = "0.4.3"
|
||||
md-5 = "0.11.0"
|
||||
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
sha2 = "0.11.0"
|
||||
tar = "0.4.46"
|
||||
tempfile = "3.27.0"
|
||||
tokio = { version = "1.52.3", features = ["fs", "io-util", "macros", "process", "rt-multi-thread", "signal", "sync", "time"] }
|
||||
walkdir = "2.5.0"
|
||||
zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
|
||||
@@ -0,0 +1,539 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::app_wasm::resolve_app_dir;
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use clap::Args;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::ffi::OsStr;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tokio::process::{Child, Command};
|
||||
use tokio::sync::watch;
|
||||
use tokio::time::timeout;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
const DEFAULT_SKIP_DIRS: &[&str] = &[".git", "node_modules", "dist", "target", "pkg", "pkgs"];
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct AppDevServerArgs {
|
||||
#[arg(long)]
|
||||
app_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct StepMetadata {
|
||||
last_run: f64,
|
||||
inputs: BTreeMap<String, f64>,
|
||||
}
|
||||
|
||||
type Metadata = BTreeMap<String, StepMetadata>;
|
||||
|
||||
pub async fn run(args: AppDevServerArgs) -> Result<()> {
|
||||
let project_root = args.app_dir.unwrap_or(resolve_app_dir()?);
|
||||
let mut server = AppDevServer::new(project_root);
|
||||
server.run().await
|
||||
}
|
||||
|
||||
struct AppDevServer {
|
||||
project_root: PathBuf,
|
||||
metadata_file: PathBuf,
|
||||
metadata: Metadata,
|
||||
}
|
||||
|
||||
impl AppDevServer {
|
||||
fn new(project_root: PathBuf) -> Self {
|
||||
Self {
|
||||
metadata_file: project_root.join(".devserver-cache.json"),
|
||||
project_root,
|
||||
metadata: Metadata::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(&mut self) -> Result<()> {
|
||||
self.load_metadata();
|
||||
let (shutdown_tx, mut shutdown_rx) = watch::channel(false);
|
||||
tokio::spawn(listen_for_shutdown(shutdown_tx));
|
||||
|
||||
self.run_cached_step(
|
||||
"wasm",
|
||||
gather_wasm_inputs,
|
||||
"pnpm wasm:codegen",
|
||||
|server, shutdown| Box::pin(server.run_command("pnpm", &["wasm:codegen"], shutdown)),
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"colors",
|
||||
gather_color_inputs,
|
||||
"pnpm generate:colors",
|
||||
|server, shutdown| Box::pin(server.run_command("pnpm", &["generate:colors"], shutdown)),
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"messageLayout",
|
||||
gather_message_layout_inputs,
|
||||
"pnpm generate:message-layout",
|
||||
|server, shutdown| {
|
||||
Box::pin(server.run_command("pnpm", &["generate:message-layout"], shutdown))
|
||||
},
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"masks",
|
||||
gather_mask_inputs,
|
||||
"pnpm generate:masks",
|
||||
|server, shutdown| Box::pin(server.run_command("pnpm", &["generate:masks"], shutdown)),
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"cssTypes",
|
||||
gather_css_module_inputs,
|
||||
"pnpm generate:css-types",
|
||||
|server, shutdown| {
|
||||
Box::pin(server.run_command("pnpm", &["generate:css-types"], shutdown))
|
||||
},
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if env_truthy("FLUXER_APP_SKIP_I18N_COMPILE") {
|
||||
eprintln!("Skipping pnpm lingui:compile because FLUXER_APP_SKIP_I18N_COMPILE is set.");
|
||||
} else {
|
||||
self.run_cached_step(
|
||||
"lingui",
|
||||
gather_lingui_inputs,
|
||||
"pnpm lingui:compile",
|
||||
|server, shutdown| {
|
||||
Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown))
|
||||
},
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if *shutdown_rx.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.clean_dist()?;
|
||||
let mut css_type_watcher = self.start_css_type_watcher()?;
|
||||
let rspack_result = self.run_rspack(&mut shutdown_rx).await;
|
||||
terminate_child(&mut css_type_watcher).await;
|
||||
rspack_result
|
||||
}
|
||||
|
||||
fn load_metadata(&mut self) {
|
||||
match fs::read_to_string(&self.metadata_file) {
|
||||
Ok(raw) => match serde_json::from_str::<Metadata>(&raw) {
|
||||
Ok(metadata) => {
|
||||
self.metadata = metadata;
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!(
|
||||
"Failed to parse dev server metadata cache, falling back to full rebuild: {error}"
|
||||
);
|
||||
self.metadata = Metadata::default();
|
||||
}
|
||||
},
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
self.metadata = Metadata::default();
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!(
|
||||
"Failed to read dev server metadata cache, falling back to full rebuild: {error}"
|
||||
);
|
||||
self.metadata = Metadata::default();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn save_metadata(&self) -> Result<()> {
|
||||
if let Some(parent) = self.metadata_file.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
fs::write(
|
||||
&self.metadata_file,
|
||||
serde_json::to_string_pretty(&self.metadata)?,
|
||||
)
|
||||
.with_context(|| format!("Failed to write {}", self.metadata_file.display()))
|
||||
}
|
||||
|
||||
async fn run_cached_step<G, E>(
|
||||
&mut self,
|
||||
step_name: &'static str,
|
||||
gather_inputs: G,
|
||||
label: &'static str,
|
||||
execute: E,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<()>
|
||||
where
|
||||
G: Fn(&Path) -> Result<BTreeMap<String, f64>>,
|
||||
E: for<'a> FnOnce(
|
||||
&'a AppDevServer,
|
||||
&'a mut watch::Receiver<bool>,
|
||||
)
|
||||
-> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>>,
|
||||
{
|
||||
let inputs = gather_inputs(&self.project_root)?;
|
||||
if !self.should_run_step(step_name, &inputs) {
|
||||
println!("Skipping {label} (no changes detected)");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
execute(self, shutdown).await?;
|
||||
self.metadata.insert(
|
||||
step_name.to_string(),
|
||||
StepMetadata {
|
||||
last_run: timestamp_ms(SystemTime::now())?,
|
||||
inputs,
|
||||
},
|
||||
);
|
||||
self.save_metadata()
|
||||
}
|
||||
|
||||
fn should_run_step(&self, step_name: &str, inputs: &BTreeMap<String, f64>) -> bool {
|
||||
let Some(entry) = self.metadata.get(step_name) else {
|
||||
return true;
|
||||
};
|
||||
&entry.inputs != inputs
|
||||
}
|
||||
|
||||
async fn run_command(
|
||||
&self,
|
||||
command: &str,
|
||||
args: &[&str],
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<()> {
|
||||
if *shutdown.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut child = spawn_child(command, args, &self.project_root)?;
|
||||
let status = wait_for_child(command, args, &mut child, shutdown).await?;
|
||||
if *shutdown.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
status.success(),
|
||||
"{} exited with status {}",
|
||||
display_command(command, args),
|
||||
status.code().unwrap_or(1)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn clean_dist(&self) -> Result<()> {
|
||||
let dist_path = self.project_root.join("dist");
|
||||
match fs::remove_dir_all(&dist_path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => {
|
||||
Err(error).with_context(|| format!("Failed to remove {}", dist_path.display()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn start_css_type_watcher(&self) -> Result<Child> {
|
||||
let tcm = self.project_root.join("node_modules/.bin/tcm");
|
||||
println!(
|
||||
"+ {} src --pattern '**/*.module.css' --watch --silent",
|
||||
tcm.display()
|
||||
);
|
||||
Command::new(tcm)
|
||||
.args(["src", "--pattern", "**/*.module.css", "--watch", "--silent"])
|
||||
.current_dir(&self.project_root)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.context("Failed to start CSS type watcher")
|
||||
}
|
||||
|
||||
async fn run_rspack(&self, shutdown: &mut watch::Receiver<bool>) -> Result<()> {
|
||||
let rspack = self.project_root.join("node_modules/.bin/rspack");
|
||||
let rspack_string = rspack.to_string_lossy().to_string();
|
||||
let mut child = spawn_child(
|
||||
&rspack_string,
|
||||
&["serve", "--mode", "development"],
|
||||
&self.project_root,
|
||||
)?;
|
||||
let status = wait_for_child(
|
||||
&rspack_string,
|
||||
&["serve", "--mode", "development"],
|
||||
&mut child,
|
||||
shutdown,
|
||||
)
|
||||
.await?;
|
||||
if *shutdown.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
bail!(
|
||||
"rspack serve exited with status {}",
|
||||
status.code().unwrap_or(1)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap<String, f64>> {
|
||||
let mut result = BTreeMap::new();
|
||||
for rel_path in paths {
|
||||
let absolute_path = project_root.join(rel_path);
|
||||
let metadata = fs::metadata(&absolute_path)
|
||||
.with_context(|| format!("Failed to stat {}", absolute_path.display()))?;
|
||||
ensure!(
|
||||
metadata.is_file(),
|
||||
"Expected {} to be a file when collecting dev server cache inputs.",
|
||||
rel_path.display()
|
||||
);
|
||||
result.insert(rel_path_key(rel_path), timestamp_ms(metadata.modified()?)?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn collect_directory_stats<P>(
|
||||
project_root: &Path,
|
||||
root_rel: &Path,
|
||||
predicate: P,
|
||||
) -> Result<BTreeMap<String, f64>>
|
||||
where
|
||||
P: Fn(&str) -> bool,
|
||||
{
|
||||
let skip_dirs: BTreeSet<&str> = DEFAULT_SKIP_DIRS.iter().copied().collect();
|
||||
let root = project_root.join(root_rel);
|
||||
let mut result = BTreeMap::new();
|
||||
if !root.exists() {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
for entry in WalkDir::new(&root)
|
||||
.into_iter()
|
||||
.filter_entry(|entry| should_walk_entry(entry.path(), &skip_dirs))
|
||||
{
|
||||
let entry = entry.with_context(|| format!("Failed to read {}", root.display()))?;
|
||||
if !entry.file_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
let rel_from_root = entry
|
||||
.path()
|
||||
.strip_prefix(&root)
|
||||
.with_context(|| format!("Failed to relativize {}", entry.path().display()))?
|
||||
.to_path_buf();
|
||||
let rel_path = root_rel.join(rel_from_root);
|
||||
let key = rel_path_key(&rel_path);
|
||||
if !predicate(&key) {
|
||||
continue;
|
||||
}
|
||||
result.insert(key, timestamp_ms(entry.metadata()?.modified()?)?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn should_walk_entry(path: &Path, skip_dirs: &BTreeSet<&str>) -> bool {
|
||||
path.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.is_none_or(|name| !skip_dirs.contains(name))
|
||||
}
|
||||
|
||||
fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
let markdown_parser_rust_dir = PathBuf::from("../packages/markdown_parser/rust");
|
||||
let mut inputs = collect_file_stats(
|
||||
project_root,
|
||||
&[
|
||||
PathBuf::from("../tools/ci/Cargo.toml"),
|
||||
PathBuf::from("../tools/ci/src/app_dev_server.rs"),
|
||||
PathBuf::from("../tools/ci/src/app_wasm.rs"),
|
||||
PathBuf::from("../tools/ci/src/common.rs"),
|
||||
PathBuf::from("../tools/ci/src/lib.rs"),
|
||||
PathBuf::from("../tools/ci/templates/libfluxcore_wrapper.js"),
|
||||
PathBuf::from("../tools/ci/templates/libfluxcore_wrapper.d.ts"),
|
||||
markdown_parser_rust_dir.join("Cargo.toml"),
|
||||
],
|
||||
)?;
|
||||
inputs.extend(collect_directory_stats(
|
||||
project_root,
|
||||
Path::new("rust/libfluxcore"),
|
||||
|path| !path.contains("/target/"),
|
||||
)?);
|
||||
inputs.extend(collect_directory_stats(
|
||||
project_root,
|
||||
&markdown_parser_rust_dir,
|
||||
|path| !path.contains("/target/"),
|
||||
)?);
|
||||
Ok(inputs)
|
||||
}
|
||||
|
||||
fn gather_color_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_file_stats(
|
||||
project_root,
|
||||
&[PathBuf::from("scripts/GenerateColorSystem.ts")],
|
||||
)
|
||||
}
|
||||
|
||||
fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_file_stats(
|
||||
project_root,
|
||||
&[
|
||||
PathBuf::from("scripts/GenerateMessageLayoutCss.ts"),
|
||||
PathBuf::from("src/features/theme/layout/MessageLayoutSpec.ts"),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_file_stats(
|
||||
project_root,
|
||||
&[
|
||||
PathBuf::from("scripts/GenerateAvatarMasks.ts"),
|
||||
PathBuf::from("src/features/ui/constants/TypingConstants.ts"),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
fn gather_css_module_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_directory_stats(project_root, Path::new("src"), |path| {
|
||||
path.ends_with(".module.css")
|
||||
})
|
||||
}
|
||||
|
||||
fn gather_lingui_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_directory_stats(
|
||||
project_root,
|
||||
Path::new("src/features/i18n/locales"),
|
||||
|path| path.ends_with(".po"),
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_child(command: &str, args: &[&str], cwd: &Path) -> Result<Child> {
|
||||
println!("+ {}", display_command(command, args));
|
||||
Command::new(command)
|
||||
.args(args)
|
||||
.current_dir(cwd)
|
||||
.stdin(Stdio::inherit())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.with_context(|| format!("Failed to run {}", display_command(command, args)))
|
||||
}
|
||||
|
||||
async fn wait_for_child(
|
||||
command: &str,
|
||||
args: &[&str],
|
||||
child: &mut Child,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<std::process::ExitStatus> {
|
||||
tokio::select! {
|
||||
status = child.wait() => {
|
||||
status.with_context(|| format!("Failed to wait for {}", display_command(command, args)))
|
||||
}
|
||||
changed = shutdown.changed() => {
|
||||
let _ = changed;
|
||||
terminate_child(child).await;
|
||||
child.wait().await.with_context(|| format!("Failed to wait for {}", display_command(command, args)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn terminate_child(child: &mut Child) {
|
||||
if child.id().is_none() {
|
||||
return;
|
||||
}
|
||||
let _ = child.start_kill();
|
||||
let _ = timeout(Duration::from_secs(5), child.wait()).await;
|
||||
}
|
||||
|
||||
async fn listen_for_shutdown(shutdown_tx: watch::Sender<bool>) {
|
||||
let signal = wait_for_shutdown_signal().await;
|
||||
println!("\nReceived {signal}, shutting down fluxer app dev server...");
|
||||
let _ = shutdown_tx.send(true);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn wait_for_shutdown_signal() -> &'static str {
|
||||
let mut sigterm = match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
{
|
||||
Ok(signal) => signal,
|
||||
Err(_) => {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
return "SIGINT";
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => "SIGINT",
|
||||
_ = sigterm.recv() => "SIGTERM",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn wait_for_shutdown_signal() -> &'static str {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
"SIGINT"
|
||||
}
|
||||
|
||||
fn rel_path_key(path: &Path) -> String {
|
||||
path.to_string_lossy().replace('\\', "/")
|
||||
}
|
||||
|
||||
fn timestamp_ms(timestamp: SystemTime) -> Result<f64> {
|
||||
Ok(timestamp
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.context("File timestamp predates UNIX epoch")?
|
||||
.as_secs_f64()
|
||||
* 1000.0)
|
||||
}
|
||||
|
||||
fn env_truthy(name: &str) -> bool {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true"))
|
||||
}
|
||||
|
||||
fn display_command(command: &str, args: &[&str]) -> String {
|
||||
std::iter::once(command.to_string())
|
||||
.chain(args.iter().map(|arg| quote_arg(arg)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
fn quote_arg(arg: &str) -> String {
|
||||
if arg
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.' | '/' | ':' | '='))
|
||||
{
|
||||
arg.to_string()
|
||||
} else {
|
||||
format!("{arg:?}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rel_path_keys_are_posix_like() {
|
||||
assert_eq!(
|
||||
rel_path_key(Path::new("scripts/GenerateColorSystem.ts")),
|
||||
"scripts/GenerateColorSystem.ts"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_command_quotes_globs() {
|
||||
assert_eq!(
|
||||
display_command("tcm", &["src", "--pattern", "**/*.module.css"]),
|
||||
"tcm src --pattern \"**/*.module.css\""
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{
|
||||
CALVER_SCHEME, CalverEnv, CommandSpec, S3UploadPlanItem, append_github_env,
|
||||
append_github_output, collect_files, path_to_s3_key, require_env, resolve_calver, run_command,
|
||||
runner_temp, s3_client, trim_option, upload_s3_plan_append_only,
|
||||
};
|
||||
use anyhow::{Context, Result, anyhow, ensure};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use chrono::Utc;
|
||||
use clap::{Args, ValueEnum};
|
||||
use serde_json::{Map, Value, json};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const DEFAULT_PUBLIC_ASSET_BASE_URL: &str = "https://fluxerstatic.com";
|
||||
const DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED: &str = "true";
|
||||
const DEFAULT_STATIC_BUCKET: &str = "fluxer-static";
|
||||
const DEFAULT_S3_ENDPOINT: &str = "https://ewr1.vultrobjects.com";
|
||||
const IMMUTABLE_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable";
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildAppProxyArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: AppProxyStep,
|
||||
#[arg(long)]
|
||||
build_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
enum AppProxyStep {
|
||||
SetMetadata,
|
||||
PrepareDockerConfig,
|
||||
ConfigureGhcrAuth,
|
||||
BuildAndExtract,
|
||||
GenerateAssetManifest,
|
||||
UploadAssets,
|
||||
}
|
||||
|
||||
pub async fn run(args: BuildAppProxyArgs) -> Result<()> {
|
||||
match args.step {
|
||||
AppProxyStep::SetMetadata => set_metadata_step(args.build_version.as_deref()),
|
||||
AppProxyStep::PrepareDockerConfig => prepare_docker_config_step(),
|
||||
AppProxyStep::ConfigureGhcrAuth => configure_ghcr_auth_step(),
|
||||
AppProxyStep::BuildAndExtract => build_and_extract_step(),
|
||||
AppProxyStep::GenerateAssetManifest => generate_asset_manifest_step(),
|
||||
AppProxyStep::UploadAssets => upload_assets_step().await,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_metadata_step(build_version_arg: Option<&str>) -> Result<()> {
|
||||
let calver_env = CalverEnv {
|
||||
build_version: trim_option(build_version_arg.map(ToOwned::to_owned))
|
||||
.or_else(|| trim_option(env::var("BUILD_VERSION").ok())),
|
||||
fluxer_build_version: trim_option(env::var("FLUXER_BUILD_VERSION").ok()),
|
||||
fluxer_build_date: trim_option(env::var("FLUXER_BUILD_DATE").ok()),
|
||||
};
|
||||
let version = resolve_calver(&calver_env, Utc::now())?;
|
||||
append_github_output(&[
|
||||
("build_version", version.as_str()),
|
||||
("version", version.as_str()),
|
||||
("calver_scheme", CALVER_SCHEME),
|
||||
])
|
||||
}
|
||||
|
||||
fn prepare_docker_config_step() -> Result<()> {
|
||||
let docker_config = runner_temp().join("docker-config");
|
||||
fs::create_dir_all(&docker_config)
|
||||
.with_context(|| format!("Failed to create {}", docker_config.display()))?;
|
||||
append_github_env(&[("DOCKER_CONFIG", docker_config.to_string_lossy().as_ref())])
|
||||
}
|
||||
|
||||
fn configure_ghcr_auth_step() -> Result<()> {
|
||||
let docker_config = require_env("DOCKER_CONFIG")?;
|
||||
let username = require_env("GHCR_USERNAME")?;
|
||||
let token = require_env("GHCR_TOKEN")?;
|
||||
let path = PathBuf::from(docker_config).join("config.json");
|
||||
write_ghcr_auth_config(&path, &username, &token)
|
||||
}
|
||||
|
||||
fn write_ghcr_auth_config(path: &Path, username: &str, token: &str) -> Result<()> {
|
||||
let mut config = if path.exists() {
|
||||
serde_json::from_str::<Value>(
|
||||
&fs::read_to_string(path)
|
||||
.with_context(|| format!("Failed to read {}", path.display()))?,
|
||||
)
|
||||
.with_context(|| format!("Failed to parse {}", path.display()))?
|
||||
} else {
|
||||
Value::Object(Map::new())
|
||||
};
|
||||
|
||||
let root = config
|
||||
.as_object_mut()
|
||||
.ok_or_else(|| anyhow!("Docker config root must be a JSON object"))?;
|
||||
let auths = root
|
||||
.entry("auths")
|
||||
.or_insert_with(|| Value::Object(Map::new()))
|
||||
.as_object_mut()
|
||||
.ok_or_else(|| anyhow!("Docker config auths must be a JSON object"))?;
|
||||
auths.insert(
|
||||
"ghcr.io".to_string(),
|
||||
json!({ "auth": ghcr_auth_value(username, token) }),
|
||||
);
|
||||
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
fs::write(path, format!("{}\n", serde_json::to_string(&config)?))
|
||||
.with_context(|| format!("Failed to write {}", path.display()))
|
||||
}
|
||||
|
||||
fn ghcr_auth_value(username: &str, token: &str) -> String {
|
||||
BASE64.encode(format!("{username}:{token}"))
|
||||
}
|
||||
|
||||
fn build_and_extract_step() -> Result<()> {
|
||||
run_command(build_and_extract_command()?)
|
||||
}
|
||||
|
||||
fn build_and_extract_command() -> Result<CommandSpec> {
|
||||
let build_version = require_env("BUILD_VERSION")?;
|
||||
let public_asset_base_url = env::var("PUBLIC_ASSET_BASE_URL")
|
||||
.unwrap_or_else(|_| DEFAULT_PUBLIC_ASSET_BASE_URL.to_string());
|
||||
let image_repo = match env::var("IMAGE_REPO") {
|
||||
Ok(value) => value,
|
||||
Err(_) => format!("ghcr.io/{}/fluxer-app-proxy", ghcr_owner()?),
|
||||
};
|
||||
Ok(CommandSpec::new("docker")
|
||||
.args(["buildx", "bake", "-f", "fluxer_app_proxy/docker-bake.hcl"])
|
||||
.env("IMAGE_REPO", image_repo)
|
||||
.env("BUILD_VERSION", build_version)
|
||||
.env("PUBLIC_ASSET_BASE_URL", public_asset_base_url)
|
||||
.env(
|
||||
"FLUXER_APP_PROXY_TIME_FREEZE_ENABLED",
|
||||
env::var("FLUXER_APP_PROXY_TIME_FREEZE_ENABLED")
|
||||
.unwrap_or_else(|_| DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED.to_string()),
|
||||
)
|
||||
.env(
|
||||
"CACHE_FROM",
|
||||
env::var("CACHE_FROM")
|
||||
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy".to_string()),
|
||||
)
|
||||
.env(
|
||||
"CACHE_TO",
|
||||
env::var("CACHE_TO")
|
||||
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy,mode=max".to_string()),
|
||||
)
|
||||
.env(
|
||||
"DOCKER_BUILD_SUMMARY",
|
||||
env::var("DOCKER_BUILD_SUMMARY").unwrap_or_else(|_| "false".to_string()),
|
||||
)
|
||||
.env(
|
||||
"DOCKER_BUILD_RECORD_UPLOAD",
|
||||
env::var("DOCKER_BUILD_RECORD_UPLOAD").unwrap_or_else(|_| "false".to_string()),
|
||||
))
|
||||
}
|
||||
|
||||
fn ghcr_owner() -> Result<String> {
|
||||
for key in ["GHCR_OWNER", "GITHUB_REPOSITORY_OWNER", "OWNER"] {
|
||||
if let Ok(value) = env::var(key) {
|
||||
let value = value.trim();
|
||||
if !value.is_empty() {
|
||||
return Ok(value.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(repository) = env::var("GITHUB_REPOSITORY") {
|
||||
if let Some((owner, _)) = repository.split_once('/') {
|
||||
let owner = owner.trim();
|
||||
if !owner.is_empty() {
|
||||
return Ok(owner.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow!(
|
||||
"GHCR owner must be set with GHCR_OWNER, GITHUB_REPOSITORY_OWNER, OWNER, or GITHUB_REPOSITORY"
|
||||
))
|
||||
}
|
||||
|
||||
fn generate_asset_manifest_step() -> Result<()> {
|
||||
let dist = app_dist_dir();
|
||||
let manifest_path = dist.join("assets-manifest.txt");
|
||||
let assets = asset_manifest_entries(&dist)?;
|
||||
fs::write(&manifest_path, format!("{}\n", assets.join("\n")))
|
||||
.with_context(|| format!("Failed to write {}", manifest_path.display()))?;
|
||||
println!("=== asset manifest ===");
|
||||
for asset in &assets {
|
||||
println!("{asset}");
|
||||
}
|
||||
println!("total assets: {}", assets.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn app_dist_dir() -> PathBuf {
|
||||
env::var("APP_DIST_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from("app-dist-output/dist"))
|
||||
}
|
||||
|
||||
fn asset_manifest_entries(dist: &Path) -> Result<Vec<String>> {
|
||||
let assets_dir = dist.join("assets");
|
||||
ensure!(
|
||||
assets_dir.exists(),
|
||||
"App proxy assets directory is missing: {}",
|
||||
assets_dir.display()
|
||||
);
|
||||
let mut entries = collect_files(&assets_dir)?
|
||||
.into_iter()
|
||||
.filter(|path| !is_source_map_asset(path))
|
||||
.map(|path| {
|
||||
path.strip_prefix(dist)
|
||||
.with_context(|| format!("Failed to relativize {}", path.display()))
|
||||
.map(path_to_s3_key)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
entries.sort();
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
fn is_source_map_asset(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|ext| ext.to_str())
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("map"))
|
||||
}
|
||||
|
||||
async fn upload_assets_step() -> Result<()> {
|
||||
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
|
||||
let bucket = env::var("STATIC_BUCKET").unwrap_or_else(|_| DEFAULT_STATIC_BUCKET.to_string());
|
||||
let dist = app_dist_dir();
|
||||
let manifest_path = dist.join("assets-manifest.txt");
|
||||
let assets = read_asset_manifest(&manifest_path)?;
|
||||
ensure!(!assets.is_empty(), "{} is empty", manifest_path.display());
|
||||
|
||||
let plan = asset_upload_plan(&dist, &assets)?;
|
||||
let stats = upload_s3_plan_append_only(&client, &bucket, plan).await?;
|
||||
|
||||
println!("upload complete - {} assets", assets.len());
|
||||
println!(
|
||||
"append-only result - uploaded {}, skipped existing {}, repaired metadata {}",
|
||||
stats.uploaded, stats.skipped_existing, stats.metadata_repaired
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn asset_upload_plan(dist: &Path, assets: &[String]) -> Result<Vec<S3UploadPlanItem>> {
|
||||
assets
|
||||
.iter()
|
||||
.map(|asset| {
|
||||
let path = dist.join(asset);
|
||||
ensure!(
|
||||
path.is_file(),
|
||||
"Manifest asset is missing: {}",
|
||||
path.display()
|
||||
);
|
||||
Ok(S3UploadPlanItem::new(path, asset.clone())
|
||||
.with_detected_content_type()
|
||||
.with_cache_control(IMMUTABLE_ASSET_CACHE_CONTROL)
|
||||
.repair_existing_metadata())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn read_asset_manifest(path: &Path) -> Result<Vec<String>> {
|
||||
let manifest =
|
||||
fs::read_to_string(path).with_context(|| format!("Failed to read {}", path.display()))?;
|
||||
manifest
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(validate_manifest_asset)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn validate_manifest_asset(asset: &str) -> Result<String> {
|
||||
ensure!(
|
||||
asset.starts_with("assets/"),
|
||||
"Asset manifest entry must be under assets/: {asset}"
|
||||
);
|
||||
ensure!(
|
||||
!asset.contains("..") && !asset.starts_with('/') && !asset.contains('\\'),
|
||||
"Invalid asset manifest path: {asset}"
|
||||
);
|
||||
Ok(asset.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::common::parse_version_instant;
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use std::ffi::OsString;
|
||||
|
||||
fn dt(year: i32, month: u32, day: u32, hour: u32, minute: u32, second: u32) -> DateTime<Utc> {
|
||||
Utc.with_ymd_and_hms(year, month, day, hour, minute, second)
|
||||
.single()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn write_file(path: &Path, contents: &str) {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_calver_from_explicit_or_date_override() {
|
||||
let explicit = CalverEnv {
|
||||
build_version: Some("2026.520.1".to_string()),
|
||||
fluxer_build_version: Some("2026.521.2".to_string()),
|
||||
fluxer_build_date: Some("2026-05-22T03:04:05Z".to_string()),
|
||||
};
|
||||
assert_eq!(
|
||||
resolve_calver(&explicit, dt(2026, 1, 1, 0, 0, 0)).unwrap(),
|
||||
"2026.520.1"
|
||||
);
|
||||
|
||||
let generated = CalverEnv {
|
||||
fluxer_build_date: Some("2026-05-20T01:02:03Z".to_string()),
|
||||
..CalverEnv::default()
|
||||
};
|
||||
assert_eq!(
|
||||
resolve_calver(&generated, dt(2026, 1, 1, 0, 0, 0)).unwrap(),
|
||||
"2026.520.10203"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_calver_time() {
|
||||
assert_eq!(
|
||||
parse_version_instant("2026.520.246000")
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"Invalid build version date/time: 2026.520.246000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghcr_auth_config_merges_existing_auths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let config_path = temp.path().join("config.json");
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"auths":{"example.com":{"auth":"old"}},"currentContext":"builder"}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
write_ghcr_auth_config(&config_path, "octo", "secret").unwrap();
|
||||
|
||||
let config: Value =
|
||||
serde_json::from_str(&fs::read_to_string(config_path).unwrap()).unwrap();
|
||||
assert_eq!(config["auths"]["example.com"]["auth"], "old");
|
||||
assert_eq!(
|
||||
config["auths"]["ghcr.io"]["auth"],
|
||||
BASE64.encode("octo:secret")
|
||||
);
|
||||
assert_eq!(config["currentContext"], "builder");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghcr_auth_config_rejects_non_object_roots_and_auths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let config_path = temp.path().join("config.json");
|
||||
fs::write(&config_path, "[]").unwrap();
|
||||
assert_eq!(
|
||||
write_ghcr_auth_config(&config_path, "octo", "secret")
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"Docker config root must be a JSON object"
|
||||
);
|
||||
|
||||
fs::write(&config_path, r#"{"auths":[]}"#).unwrap();
|
||||
assert_eq!(
|
||||
write_ghcr_auth_config(&config_path, "octo", "secret")
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"Docker config auths must be a JSON object"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_command_sets_bake_environment() {
|
||||
let command = CommandSpec::new("docker")
|
||||
.args(["buildx", "bake", "-f", "fluxer_app_proxy/docker-bake.hcl"])
|
||||
.env("IMAGE_REPO", "ghcr.io/example/fluxer-app-proxy")
|
||||
.env("BUILD_VERSION", "2026.520.1")
|
||||
.env("PUBLIC_ASSET_BASE_URL", DEFAULT_PUBLIC_ASSET_BASE_URL)
|
||||
.env(
|
||||
"FLUXER_APP_PROXY_TIME_FREEZE_ENABLED",
|
||||
DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED,
|
||||
);
|
||||
|
||||
assert_eq!(command.program, OsString::from("docker"));
|
||||
assert_eq!(
|
||||
command.args,
|
||||
vec![
|
||||
OsString::from("buildx"),
|
||||
OsString::from("bake"),
|
||||
OsString::from("-f"),
|
||||
OsString::from("fluxer_app_proxy/docker-bake.hcl"),
|
||||
]
|
||||
);
|
||||
assert!(command.env.contains(&(
|
||||
OsString::from("BUILD_VERSION"),
|
||||
OsString::from("2026.520.1")
|
||||
)));
|
||||
assert!(command.env.contains(&(
|
||||
OsString::from("FLUXER_APP_PROXY_TIME_FREEZE_ENABLED"),
|
||||
OsString::from(DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED)
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_manifest_entries_are_sorted_and_relative_to_dist() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
write_file(&dist.join("assets/z.js"), "z");
|
||||
write_file(&dist.join("assets/z.js.map"), "{}");
|
||||
write_file(&dist.join("assets/chunks/a.js"), "a");
|
||||
write_file(&dist.join("assets/chunks/a.js.map"), "{}");
|
||||
write_file(&dist.join("index.html"), "ignored");
|
||||
|
||||
assert_eq!(
|
||||
asset_manifest_entries(&dist).unwrap(),
|
||||
vec!["assets/chunks/a.js", "assets/z.js"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_manifest_entries_require_assets_directory() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
fs::create_dir_all(&dist).unwrap();
|
||||
|
||||
assert!(
|
||||
asset_manifest_entries(&dist)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("App proxy assets directory is missing")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_reader_trims_blank_lines_and_keeps_order() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let manifest = temp.path().join("assets-manifest.txt");
|
||||
fs::write(&manifest, "\n assets/b.js \n\nassets/a.js\n").unwrap();
|
||||
|
||||
assert_eq!(
|
||||
read_asset_manifest(&manifest).unwrap(),
|
||||
vec!["assets/b.js", "assets/a.js"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_upload_plan_preserves_manifest_keys() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
write_file(&dist.join("assets/a.js"), "a");
|
||||
write_file(&dist.join("assets/chunks/b.js"), "b");
|
||||
let assets = vec!["assets/a.js".to_string(), "assets/chunks/b.js".to_string()];
|
||||
|
||||
let plan = asset_upload_plan(&dist, &assets).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
plan.iter()
|
||||
.map(|item| item.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["assets/a.js", "assets/chunks/b.js"]
|
||||
);
|
||||
assert_eq!(plan[0].path, dist.join("assets/a.js"));
|
||||
assert_eq!(plan[1].path, dist.join("assets/chunks/b.js"));
|
||||
assert_eq!(
|
||||
plan[0].content_type.as_deref(),
|
||||
Some("application/javascript; charset=utf-8")
|
||||
);
|
||||
assert_eq!(
|
||||
plan[0].cache_control.as_deref(),
|
||||
Some(IMMUTABLE_ASSET_CACHE_CONTROL)
|
||||
);
|
||||
assert!(plan[0].repair_existing_metadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_upload_plan_rejects_manifest_entries_missing_on_disk() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
fs::create_dir_all(&dist).unwrap();
|
||||
let assets = vec!["assets/missing.js".to_string()];
|
||||
|
||||
assert!(
|
||||
asset_upload_plan(&dist, &assets)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("Manifest asset is missing")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_reader_rejects_paths_outside_assets() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let manifest = temp.path().join("assets-manifest.txt");
|
||||
fs::write(&manifest, "assets/a.js\n../secret\n").unwrap();
|
||||
|
||||
assert!(read_asset_manifest(&manifest).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_reader_rejects_absolute_parent_and_backslash_paths() {
|
||||
for asset in ["/assets/a.js", "assets/../secret", r"assets\app.js"] {
|
||||
assert!(validate_manifest_asset(asset).is_err(), "{asset}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_to_s3_key_uses_forward_slashes() {
|
||||
assert_eq!(
|
||||
path_to_s3_key(Path::new("assets").join("chunks").join("a.js").as_path()),
|
||||
"assets/chunks/a.js"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{CommandSpec, command_succeeds, env_bool, output_text, run_command};
|
||||
use anyhow::{Context, Result, anyhow, ensure};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use clap::Args;
|
||||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tempfile::TempDir;
|
||||
|
||||
const LIBFLUXCORE_WASM_BINDGEN_VERSION: &str = "0.2.122";
|
||||
const LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES: u64 = 300 * 1024;
|
||||
const LIBFLUXCORE_WRAPPER_JS: &str = include_str!("../templates/libfluxcore_wrapper.js");
|
||||
const LIBFLUXCORE_WRAPPER_DTS: &str = include_str!("../templates/libfluxcore_wrapper.d.ts");
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildAppWasmArgs {
|
||||
#[arg(long)]
|
||||
app_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildMarkdownParserWasmArgs {
|
||||
#[arg(long)]
|
||||
app_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
pub fn run_build_app_wasm(args: BuildAppWasmArgs) -> Result<()> {
|
||||
let app_dir = args.app_dir.unwrap_or(resolve_app_dir()?);
|
||||
build_markdown_parser_wasm(&app_dir)?;
|
||||
build_libfluxcore_wasm(&app_dir)
|
||||
}
|
||||
|
||||
pub fn run_build_markdown_parser_wasm(args: BuildMarkdownParserWasmArgs) -> Result<()> {
|
||||
let app_dir = args.app_dir.unwrap_or(resolve_app_dir()?);
|
||||
build_markdown_parser_wasm(&app_dir)
|
||||
}
|
||||
|
||||
fn build_markdown_parser_wasm(app_dir: &Path) -> Result<()> {
|
||||
let rust_source_dir = app_dir.join("../packages/markdown_parser/rust");
|
||||
let bytes_path =
|
||||
app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts");
|
||||
let temp = TempDir::new().context("Failed to create source temp directory")?;
|
||||
let target_dir = temp.path().join("target");
|
||||
|
||||
run_command(
|
||||
CommandSpec::new("cargo")
|
||||
.args(["build", "--release", "--target", "wasm32-unknown-unknown"])
|
||||
.env("CARGO_TARGET_DIR", target_dir.to_string_lossy().as_ref())
|
||||
.current_dir(&rust_source_dir),
|
||||
)?;
|
||||
|
||||
let wasm_path = target_dir.join("wasm32-unknown-unknown/release/fluxer_markdown_parser.wasm");
|
||||
let wasm =
|
||||
fs::read(&wasm_path).with_context(|| format!("Failed to read {}", wasm_path.display()))?;
|
||||
let content = markdown_wasm_bytes_content(&wasm);
|
||||
if let Some(parent) = bytes_path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
fs::write(&bytes_path, content)
|
||||
.with_context(|| format!("Failed to write {}", bytes_path.display()))
|
||||
}
|
||||
|
||||
fn build_libfluxcore_wasm(app_dir: &Path) -> Result<()> {
|
||||
let rust_package_dir = app_dir.join("rust/libfluxcore");
|
||||
let out_dir = app_dir.join("pkgs/libfluxcore");
|
||||
let wasm_path = out_dir.join("libfluxcore_bg.wasm");
|
||||
let previous_wasm_size = file_size(&wasm_path)?;
|
||||
|
||||
fs::create_dir_all(&out_dir)
|
||||
.with_context(|| format!("Failed to create {}", out_dir.display()))?;
|
||||
|
||||
let mut build = CommandSpec::new("cargo")
|
||||
.args([
|
||||
"build",
|
||||
"--release",
|
||||
"--target",
|
||||
"wasm32-unknown-unknown",
|
||||
"--manifest-path",
|
||||
])
|
||||
.arg(rust_package_dir.join("Cargo.toml"))
|
||||
.current_dir(&rust_package_dir);
|
||||
|
||||
if env_bool("FLUXCORE_WASM_SIMD") {
|
||||
let rustflags = match env::var("RUSTFLAGS") {
|
||||
Ok(value) if !value.trim().is_empty() => format!("{value} -C target-feature=+simd128"),
|
||||
_ => "-C target-feature=+simd128".to_string(),
|
||||
};
|
||||
build = build.env("RUSTFLAGS", rustflags);
|
||||
}
|
||||
|
||||
run_command(build)?;
|
||||
let wasm_bindgen = ensure_wasm_bindgen_cli()?;
|
||||
let temp = TempDir::new().context("Failed to create libfluxcore wasm-bindgen temp dir")?;
|
||||
let bindgen_dir = temp.path().join("bindgen");
|
||||
fs::create_dir_all(&bindgen_dir)
|
||||
.with_context(|| format!("Failed to create {}", bindgen_dir.display()))?;
|
||||
|
||||
run_command(
|
||||
CommandSpec::new(wasm_bindgen)
|
||||
.args(["--target", "web", "--out-dir"])
|
||||
.arg(&bindgen_dir)
|
||||
.args(["--out-name", "libfluxcore"])
|
||||
.arg(rust_package_dir.join("target/wasm32-unknown-unknown/release/libfluxcore.wasm"))
|
||||
.current_dir(&rust_package_dir),
|
||||
)?;
|
||||
|
||||
let bindgen_js_path = bindgen_dir.join("libfluxcore.js");
|
||||
let bindgen_dts_path = bindgen_dir.join("libfluxcore.d.ts");
|
||||
let bindgen_wasm_path = bindgen_dir.join("libfluxcore_bg.wasm");
|
||||
let bindgen_wasm_dts_path = bindgen_dir.join("libfluxcore_bg.wasm.d.ts");
|
||||
|
||||
write_with_spdx(
|
||||
&out_dir.join("libfluxcore_bindgen.js"),
|
||||
&patch_libfluxcore_bindgen_js(
|
||||
&fs::read_to_string(&bindgen_js_path)
|
||||
.with_context(|| format!("Failed to read {}", bindgen_js_path.display()))?,
|
||||
)?,
|
||||
)?;
|
||||
write_with_spdx(
|
||||
&out_dir.join("libfluxcore_bindgen.d.ts"),
|
||||
&patch_libfluxcore_bindgen_dts(
|
||||
&fs::read_to_string(&bindgen_dts_path)
|
||||
.with_context(|| format!("Failed to read {}", bindgen_dts_path.display()))?,
|
||||
)?,
|
||||
)?;
|
||||
fs::copy(&bindgen_wasm_path, &wasm_path).with_context(|| {
|
||||
format!(
|
||||
"Failed to copy {} to {}",
|
||||
bindgen_wasm_path.display(),
|
||||
wasm_path.display()
|
||||
)
|
||||
})?;
|
||||
fs::copy(
|
||||
&bindgen_wasm_dts_path,
|
||||
out_dir.join("libfluxcore_bg.wasm.d.ts"),
|
||||
)
|
||||
.with_context(|| format!("Failed to copy {}", bindgen_wasm_dts_path.display()))?;
|
||||
|
||||
fs::write(
|
||||
out_dir.join("libfluxcore.js"),
|
||||
libfluxcore_index_js_content(),
|
||||
)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to write {}",
|
||||
out_dir.join("libfluxcore.js").display()
|
||||
)
|
||||
})?;
|
||||
fs::write(
|
||||
out_dir.join("libfluxcore.d.ts"),
|
||||
libfluxcore_index_dts_content(),
|
||||
)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to write {}",
|
||||
out_dir.join("libfluxcore.d.ts").display()
|
||||
)
|
||||
})?;
|
||||
fs::write(
|
||||
out_dir.join("package.json"),
|
||||
libfluxcore_package_json_content(),
|
||||
)
|
||||
.with_context(|| format!("Failed to write {}", out_dir.join("package.json").display()))?;
|
||||
fs::write(out_dir.join("README.md"), libfluxcore_readme_content())
|
||||
.with_context(|| format!("Failed to write {}", out_dir.join("README.md").display()))?;
|
||||
|
||||
let wasm_size = file_size(&wasm_path)?
|
||||
.ok_or_else(|| anyhow!("libfluxcore build did not emit {}", wasm_path.display()))?;
|
||||
ensure!(
|
||||
wasm_size <= LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES,
|
||||
"libfluxcore_bg.wasm is {}, over the {} budget",
|
||||
format_bytes(wasm_size),
|
||||
format_bytes(LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES)
|
||||
);
|
||||
|
||||
let size_comparison = match previous_wasm_size {
|
||||
Some(previous) => format!("{} -> {}", format_bytes(previous), format_bytes(wasm_size)),
|
||||
None => "no previous artifact".to_string(),
|
||||
};
|
||||
println!(
|
||||
"libfluxcore_bg.wasm size: {size_comparison} (budget {})",
|
||||
format_bytes(LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES)
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn patch_libfluxcore_bindgen_js(content: &str) -> Result<String> {
|
||||
const MARKER: &str = "\nasync function __wbg_load(module, imports) {";
|
||||
const RESET_EXPORT: &str = r#"
|
||||
export function __resetLibfluxcoreWasmForMemoryPressure() {
|
||||
wasmModule = undefined;
|
||||
wasmInstance = undefined;
|
||||
wasm = undefined;
|
||||
cachedDataViewMemory0 = null;
|
||||
cachedUint8ArrayMemory0 = null;
|
||||
heap = new Array(1024).fill(undefined);
|
||||
heap.push(undefined, null, true, false);
|
||||
heap_next = heap.length;
|
||||
numBytesDecoded = 0;
|
||||
}
|
||||
"#;
|
||||
ensure!(
|
||||
content.contains(MARKER),
|
||||
"libfluxcore wasm-bindgen JS output did not contain reset insertion marker"
|
||||
);
|
||||
Ok(content.replacen(MARKER, &format!("{RESET_EXPORT}{MARKER}"), 1))
|
||||
}
|
||||
|
||||
fn patch_libfluxcore_bindgen_dts(content: &str) -> Result<String> {
|
||||
const MARKER: &str = "\nexport type InitInput";
|
||||
const RESET_EXPORT: &str =
|
||||
"\nexport function __resetLibfluxcoreWasmForMemoryPressure(): void;\n";
|
||||
ensure!(
|
||||
content.contains(MARKER),
|
||||
"libfluxcore wasm-bindgen DTS output did not contain reset insertion marker"
|
||||
);
|
||||
Ok(content.replacen(MARKER, &format!("{RESET_EXPORT}{MARKER}"), 1))
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_app_dir() -> Result<PathBuf> {
|
||||
let cwd = env::current_dir().context("Failed to resolve current directory")?;
|
||||
if cwd.file_name().and_then(|value| value.to_str()) == Some("fluxer_app") {
|
||||
return Ok(cwd);
|
||||
}
|
||||
if cwd.join("fluxer_app").is_dir() {
|
||||
return Ok(cwd.join("fluxer_app"));
|
||||
}
|
||||
Err(anyhow!(
|
||||
"Could not resolve fluxer_app directory from {}",
|
||||
cwd.display()
|
||||
))
|
||||
}
|
||||
|
||||
fn ensure_wasm_bindgen_cli() -> Result<OsString> {
|
||||
let expected = format!("wasm-bindgen {LIBFLUXCORE_WASM_BINDGEN_VERSION}");
|
||||
if command_succeeds(CommandSpec::new("wasm-bindgen").arg("--version")) {
|
||||
let version = output_text(CommandSpec::new("wasm-bindgen").arg("--version"))?;
|
||||
if version.trim() == expected {
|
||||
return Ok("wasm-bindgen".into());
|
||||
}
|
||||
}
|
||||
|
||||
run_command(CommandSpec::new("cargo").args([
|
||||
"install",
|
||||
"wasm-bindgen-cli",
|
||||
"--version",
|
||||
LIBFLUXCORE_WASM_BINDGEN_VERSION,
|
||||
"--locked",
|
||||
"--force",
|
||||
]))?;
|
||||
Ok("wasm-bindgen".into())
|
||||
}
|
||||
|
||||
fn file_size(path: &Path) -> Result<Option<u64>> {
|
||||
match fs::metadata(path) {
|
||||
Ok(metadata) => Ok(Some(metadata.len())),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error).with_context(|| format!("Failed to stat {}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_bytes(bytes: u64) -> String {
|
||||
format!("{bytes} B")
|
||||
}
|
||||
|
||||
fn write_with_spdx(path: &Path, content: &str) -> Result<()> {
|
||||
fs::write(
|
||||
path,
|
||||
format!("// SPDX-License-Identifier: AGPL-3.0-or-later\n{content}"),
|
||||
)
|
||||
.with_context(|| format!("Failed to write {}", path.display()))
|
||||
}
|
||||
|
||||
fn libfluxcore_index_js_content() -> String {
|
||||
format!(
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
import {{crop_rotate_rgba_raw}} from './libfluxcore_bindgen.js';\n\
|
||||
{LIBFLUXCORE_WRAPPER_JS}\n\
|
||||
export * from './libfluxcore_bindgen.js';\n\
|
||||
export {{default}} from './libfluxcore_bindgen.js';\n"
|
||||
)
|
||||
}
|
||||
|
||||
fn libfluxcore_index_dts_content() -> String {
|
||||
format!(
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
export * from './libfluxcore_bindgen.js';\n\
|
||||
export {{default}} from './libfluxcore_bindgen.js';\n\n\
|
||||
{LIBFLUXCORE_WRAPPER_DTS}"
|
||||
)
|
||||
}
|
||||
|
||||
fn libfluxcore_package_json_content() -> String {
|
||||
let manifest = serde_json::json!({
|
||||
"name": "libfluxcore",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"version": "0.0.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"sideEffects": false,
|
||||
"files": [
|
||||
"libfluxcore.js",
|
||||
"libfluxcore.d.ts",
|
||||
"libfluxcore_bindgen.js",
|
||||
"libfluxcore_bindgen.d.ts",
|
||||
"libfluxcore_bg.wasm",
|
||||
"libfluxcore_bg.wasm.d.ts",
|
||||
"README.md"
|
||||
],
|
||||
"main": "libfluxcore.js",
|
||||
"module": "libfluxcore.js",
|
||||
"types": "libfluxcore.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./libfluxcore.d.ts",
|
||||
"default": "./libfluxcore.js"
|
||||
},
|
||||
"./libfluxcore_bg.wasm": "./libfluxcore_bg.wasm"
|
||||
}
|
||||
});
|
||||
format!("{manifest:#}\n")
|
||||
}
|
||||
|
||||
fn libfluxcore_readme_content() -> &'static str {
|
||||
"<!-- SPDX-License-Identifier: AGPL-3.0-or-later -->\n\
|
||||
# libfluxcore\n\n\
|
||||
Rust WebAssembly helpers and JavaScript codec wrappers for Fluxer media processing.\n"
|
||||
}
|
||||
|
||||
fn markdown_wasm_bytes_content(wasm: &[u8]) -> String {
|
||||
format!(
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
export const MARKDOWN_PARSER_WASM_BASE64 =\n\
|
||||
\t'{}';\n",
|
||||
BASE64.encode(wasm)
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn markdown_wasm_bytes_content_matches_legacy_node_output() {
|
||||
assert_eq!(
|
||||
markdown_wasm_bytes_content(b"hello"),
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
export const MARKDOWN_PARSER_WASM_BASE64 =\n\
|
||||
\t'aGVsbG8=';\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libfluxcore_index_reexports_bindgen_module() {
|
||||
let content = libfluxcore_index_js_content();
|
||||
assert!(content.contains("import {crop_rotate_rgba_raw} from './libfluxcore_bindgen.js';"));
|
||||
assert!(content.contains("export * from './libfluxcore_bindgen.js';"));
|
||||
assert!(content.contains("export function crop_rotate_rgba("));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libfluxcore_bindgen_js_reset_hook_is_inserted() {
|
||||
let content =
|
||||
"function __wbg_finalize_init() {}\nasync function __wbg_load(module, imports) {}";
|
||||
let patched = patch_libfluxcore_bindgen_js(content).expect("patch should succeed");
|
||||
assert!(patched.contains("export function __resetLibfluxcoreWasmForMemoryPressure()"));
|
||||
assert!(patched.contains("wasm = undefined;"));
|
||||
assert!(patched.contains("async function __wbg_load(module, imports) {}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libfluxcore_bindgen_dts_reset_hook_is_inserted() {
|
||||
let content = "export function is_animated_image(input: Uint8Array): boolean;\nexport type InitInput = RequestInfo;";
|
||||
let patched = patch_libfluxcore_bindgen_dts(content).expect("patch should succeed");
|
||||
assert!(
|
||||
patched.contains("export function __resetLibfluxcoreWasmForMemoryPressure(): void;")
|
||||
);
|
||||
assert!(patched.contains("export type InitInput = RequestInfo;"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{
|
||||
CALVER_SCHEME, CalverEnv, append_github_env, append_github_output, parse_version_instant,
|
||||
resolve_calver, trim_option,
|
||||
};
|
||||
use anyhow::Result;
|
||||
use chrono::{Datelike, Timelike, Utc};
|
||||
use clap::Args;
|
||||
use std::env;
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct ResolveCalverArgs {
|
||||
#[arg(long)]
|
||||
github_output: bool,
|
||||
#[arg(long)]
|
||||
github_env: bool,
|
||||
#[arg(long, default_value = "BUILD_VERSION")]
|
||||
env_name: String,
|
||||
}
|
||||
|
||||
pub fn run(args: ResolveCalverArgs) -> Result<()> {
|
||||
let resolved = resolve_calver_from_env()?;
|
||||
if args.github_output {
|
||||
let output = calver_outputs(&resolved)?;
|
||||
append_github_output(&[
|
||||
("version", output.version.as_str()),
|
||||
("build_version", output.version.as_str()),
|
||||
("time", output.time.as_str()),
|
||||
("micro", output.micro.as_str()),
|
||||
("patch", output.micro.as_str()),
|
||||
("date", output.date.as_str()),
|
||||
("year", output.year.as_str()),
|
||||
("month", output.month.as_str()),
|
||||
("day", output.day.as_str()),
|
||||
("month_day", output.month_day.as_str()),
|
||||
("calver_scheme", CALVER_SCHEME),
|
||||
])?;
|
||||
}
|
||||
if args.github_env {
|
||||
append_github_env(&[(args.env_name.as_str(), resolved.as_str())])?;
|
||||
}
|
||||
println!("{resolved}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resolve_calver_from_env() -> Result<String> {
|
||||
resolve_calver(
|
||||
&CalverEnv {
|
||||
build_version: trim_option(env::var("BUILD_VERSION").ok()),
|
||||
fluxer_build_version: trim_option(env::var("FLUXER_BUILD_VERSION").ok()),
|
||||
fluxer_build_date: trim_option(env::var("FLUXER_BUILD_DATE").ok()),
|
||||
},
|
||||
Utc::now(),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct CalverOutputs {
|
||||
version: String,
|
||||
time: String,
|
||||
micro: String,
|
||||
date: String,
|
||||
year: String,
|
||||
month: String,
|
||||
day: String,
|
||||
month_day: String,
|
||||
}
|
||||
|
||||
fn calver_outputs(version: &str) -> Result<CalverOutputs> {
|
||||
let instant = parse_version_instant(version)?;
|
||||
let time = format!(
|
||||
"{:02}{:02}{:02}",
|
||||
instant.hour(),
|
||||
instant.minute(),
|
||||
instant.second()
|
||||
);
|
||||
let micro = time
|
||||
.parse::<u32>()
|
||||
.expect("HHMMSS time segment should parse")
|
||||
.to_string();
|
||||
Ok(CalverOutputs {
|
||||
version: version.to_string(),
|
||||
time,
|
||||
micro,
|
||||
date: format!(
|
||||
"{:04}{:02}{:02}",
|
||||
instant.year(),
|
||||
instant.month(),
|
||||
instant.day()
|
||||
),
|
||||
year: instant.year().to_string(),
|
||||
month: instant.month().to_string(),
|
||||
day: format!("{:02}", instant.day()),
|
||||
month_day: format!("{}{:02}", instant.month(), instant.day()),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::common::CalverEnv;
|
||||
use chrono::{TimeZone, Utc};
|
||||
|
||||
#[test]
|
||||
fn calver_outputs_match_legacy_shell_fields() {
|
||||
assert_eq!(
|
||||
calver_outputs("2026.520.10203").unwrap(),
|
||||
CalverOutputs {
|
||||
version: "2026.520.10203".to_string(),
|
||||
time: "010203".to_string(),
|
||||
micro: "10203".to_string(),
|
||||
date: "20260520".to_string(),
|
||||
year: "2026".to_string(),
|
||||
month: "5".to_string(),
|
||||
day: "20".to_string(),
|
||||
month_day: "520".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calver_date_only_override_matches_legacy_shell() {
|
||||
let version = resolve_calver(
|
||||
&CalverEnv {
|
||||
fluxer_build_date: Some("2026-01-09".to_string()),
|
||||
..CalverEnv::default()
|
||||
},
|
||||
Utc.with_ymd_and_hms(2026, 5, 20, 1, 2, 3).single().unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(version, "2026.109.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calver_rejects_invalid_time() {
|
||||
assert_eq!(
|
||||
calver_outputs("2026.520.246000").unwrap_err().to_string(),
|
||||
"Invalid build version date/time: 2026.520.246000"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{CommandSpec, run_command};
|
||||
use crate::gateway::{GatewayStep, run_gateway_step};
|
||||
use anyhow::{Context, Result};
|
||||
use clap::{Args, ValueEnum};
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct CiArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: CiStep,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
enum CiStep {
|
||||
InstallDependencies,
|
||||
Typecheck,
|
||||
Test,
|
||||
Knip,
|
||||
GatewayFmt,
|
||||
GatewayCompile,
|
||||
GatewayDialyzer,
|
||||
GatewayEunit,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct CiScriptsArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: CiScriptsStep,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
enum CiScriptsStep {
|
||||
Sync,
|
||||
Test,
|
||||
}
|
||||
|
||||
pub async fn run_ci(args: CiArgs) -> Result<()> {
|
||||
let root = repo_root()?;
|
||||
match args.step {
|
||||
CiStep::InstallDependencies => run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["install", "--frozen-lockfile"])
|
||||
.current_dir(root),
|
||||
),
|
||||
CiStep::Typecheck => {
|
||||
run_generators(&root, true)?;
|
||||
run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["-r", "--if-present", "typecheck"])
|
||||
.current_dir(root),
|
||||
)
|
||||
}
|
||||
CiStep::Test => {
|
||||
run_generators(&root, false)?;
|
||||
run_workspace_tests(&root)?;
|
||||
run_command(with_test_env(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["--filter", "fluxer_api", "test"])
|
||||
.current_dir(root),
|
||||
))
|
||||
}
|
||||
CiStep::Knip => {
|
||||
run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["--filter", "fluxer_app", "i18n:compile"])
|
||||
.current_dir(&root),
|
||||
)?;
|
||||
run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["exec", "knip"])
|
||||
.current_dir(root),
|
||||
)
|
||||
}
|
||||
CiStep::GatewayFmt => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::FmtCheck, "test")
|
||||
}
|
||||
CiStep::GatewayCompile => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Compile, "test")
|
||||
}
|
||||
CiStep::GatewayDialyzer => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Dialyzer, "test")
|
||||
}
|
||||
CiStep::GatewayEunit => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Eunit, "test")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_ci_scripts(args: CiScriptsArgs) -> Result<()> {
|
||||
let root = repo_root()?;
|
||||
match args.step {
|
||||
CiScriptsStep::Sync => run_command(
|
||||
CommandSpec::new("cargo")
|
||||
.args([
|
||||
"fetch",
|
||||
"--locked",
|
||||
"--manifest-path",
|
||||
"tools/ci/Cargo.toml",
|
||||
])
|
||||
.current_dir(root),
|
||||
),
|
||||
CiScriptsStep::Test => run_command(
|
||||
CommandSpec::new("cargo")
|
||||
.args(["test", "--locked", "--manifest-path", "tools/ci/Cargo.toml"])
|
||||
.current_dir(root),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_generators(root: &Path, for_typecheck: bool) -> Result<()> {
|
||||
for command in generator_commands(for_typecheck) {
|
||||
run_command(command.current_dir(root))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generator_commands(for_typecheck: bool) -> Vec<CommandSpec> {
|
||||
let mut commands = vec![
|
||||
CommandSpec::new("pnpm").args(["--filter", "@fluxer/config", "generate"]),
|
||||
CommandSpec::new("pnpm").args(["--filter", "@fluxer/schema", "generate"]),
|
||||
];
|
||||
if for_typecheck {
|
||||
commands.push(CommandSpec::new("pnpm").args([
|
||||
"--filter",
|
||||
"@fluxer/i18n",
|
||||
"generate:types",
|
||||
]));
|
||||
}
|
||||
commands.push(CommandSpec::new("pnpm").args(["--filter", "fluxer_app", "i18n:compile"]));
|
||||
commands
|
||||
}
|
||||
|
||||
fn run_workspace_tests(root: &Path) -> Result<()> {
|
||||
let workspace_concurrency =
|
||||
env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").unwrap_or_else(|_| "2".to_string());
|
||||
run_command(with_test_env(
|
||||
CommandSpec::new("pnpm")
|
||||
.args([
|
||||
"-r",
|
||||
&format!("--workspace-concurrency={workspace_concurrency}"),
|
||||
"--filter",
|
||||
"!fluxer_api",
|
||||
"--filter",
|
||||
"!fluxer",
|
||||
"--if-present",
|
||||
"test",
|
||||
])
|
||||
.current_dir(root),
|
||||
))
|
||||
}
|
||||
|
||||
fn with_test_env(spec: CommandSpec) -> CommandSpec {
|
||||
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
|
||||
let api_workers = env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| "2".to_string());
|
||||
spec.env("FLUXER_NATS_URL", &nats_url)
|
||||
.env(
|
||||
"FLUXER_NATS_CORE_URL",
|
||||
env::var("FLUXER_NATS_CORE_URL").unwrap_or_else(|_| nats_url.clone()),
|
||||
)
|
||||
.env(
|
||||
"FLUXER_NATS_JETSTREAM_URL",
|
||||
env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone()),
|
||||
)
|
||||
.env("API_TEST_MAX_WORKERS", &api_workers)
|
||||
.env(
|
||||
"API_TEST_MAX_CONCURRENCY",
|
||||
env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers),
|
||||
)
|
||||
}
|
||||
|
||||
fn default_test_nats_url() -> String {
|
||||
if Path::new("/.dockerenv").exists() {
|
||||
"nats://nats:4222".to_string()
|
||||
} else {
|
||||
"nats://127.0.0.1:4222".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_root() -> Result<PathBuf> {
|
||||
env::var("GITHUB_WORKSPACE")
|
||||
.map(PathBuf::from)
|
||||
.or_else(|_| env::current_dir())
|
||||
.context("Failed to resolve repository root")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::ffi::OsString;
|
||||
|
||||
#[test]
|
||||
fn generator_commands_include_i18n_types_only_for_typecheck() {
|
||||
let typecheck = generator_commands(true)
|
||||
.into_iter()
|
||||
.map(|command| command.args)
|
||||
.collect::<Vec<_>>();
|
||||
let test = generator_commands(false)
|
||||
.into_iter()
|
||||
.map(|command| command.args)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert!(typecheck.contains(&vec![
|
||||
OsString::from("--filter"),
|
||||
OsString::from("@fluxer/i18n"),
|
||||
OsString::from("generate:types"),
|
||||
]));
|
||||
assert!(!test.contains(&vec![
|
||||
OsString::from("--filter"),
|
||||
OsString::from("@fluxer/i18n"),
|
||||
OsString::from("generate:types"),
|
||||
]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_test_env_sets_all_nats_urls_and_concurrency() {
|
||||
let spec = with_test_env(CommandSpec::new("pnpm"));
|
||||
let env = spec
|
||||
.env
|
||||
.into_iter()
|
||||
.collect::<std::collections::BTreeMap<_, _>>();
|
||||
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("FLUXER_NATS_URL")),
|
||||
Some(&OsString::from("nats://nats:4222"))
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("FLUXER_NATS_CORE_URL")),
|
||||
Some(&OsString::from("nats://nats:4222"))
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("FLUXER_NATS_JETSTREAM_URL")),
|
||||
Some(&OsString::from("nats://nats:4222"))
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("API_TEST_MAX_WORKERS")),
|
||||
Some(&OsString::from("2"))
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("API_TEST_MAX_CONCURRENCY")),
|
||||
Some(&OsString::from("2"))
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,496 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{CommandSpec, command_succeeds, output_text, run_command};
|
||||
use anyhow::{Context, Result, anyhow, bail, ensure};
|
||||
use clap::{Args, ValueEnum};
|
||||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const GATEWAY_NIF_CRATES: &[&str] = &["push_markdown_plaintext_nif", "guild_member_list_oset_nif"];
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildGatewayNifsArgs {
|
||||
#[arg(long)]
|
||||
gateway_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct GatewayArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: GatewayStep,
|
||||
#[arg(long, default_value = "test")]
|
||||
eqwalizer_profile: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
pub(crate) enum GatewayStep {
|
||||
Fmt,
|
||||
FmtCheck,
|
||||
Lint,
|
||||
Compile,
|
||||
ProdCompile,
|
||||
Dialyzer,
|
||||
Eqwalizer,
|
||||
Typecheck,
|
||||
Eunit,
|
||||
Bench,
|
||||
AllChecks,
|
||||
Clean,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum NifBuildProfile {
|
||||
Debug,
|
||||
Release,
|
||||
}
|
||||
|
||||
impl NifBuildProfile {
|
||||
fn from_env_value(value: Option<&str>) -> Self {
|
||||
match value {
|
||||
Some("release") | None => Self::Release,
|
||||
Some(_) => Self::Debug,
|
||||
}
|
||||
}
|
||||
|
||||
fn target_dir_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Debug => "debug",
|
||||
Self::Release => "release",
|
||||
}
|
||||
}
|
||||
|
||||
fn cargo_args(self) -> Vec<OsString> {
|
||||
let mut args = vec![OsString::from("build")];
|
||||
if self == Self::Release {
|
||||
args.push(OsString::from("--release"));
|
||||
}
|
||||
args
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct GatewayNifBuild {
|
||||
crate_name: String,
|
||||
native_dir: PathBuf,
|
||||
cargo_args: Vec<OsString>,
|
||||
artifact_path: PathBuf,
|
||||
output_path: PathBuf,
|
||||
}
|
||||
|
||||
pub fn run_build_gateway_nifs(args: BuildGatewayNifsArgs) -> Result<()> {
|
||||
let gateway_dir = args
|
||||
.gateway_dir
|
||||
.map(Ok)
|
||||
.unwrap_or_else(resolve_gateway_dir)?;
|
||||
build_gateway_nifs(&gateway_dir)
|
||||
}
|
||||
|
||||
pub fn run_gateway(args: GatewayArgs) -> Result<()> {
|
||||
let gateway_dir = resolve_gateway_dir()?;
|
||||
run_gateway_step(&gateway_dir, args.step, &args.eqwalizer_profile)
|
||||
}
|
||||
|
||||
pub(crate) fn run_gateway_step(
|
||||
gateway_dir: &Path,
|
||||
step: GatewayStep,
|
||||
eqwalizer_profile: &str,
|
||||
) -> Result<()> {
|
||||
match step {
|
||||
GatewayStep::Fmt => run_rebar(gateway_dir, ["fmt"]),
|
||||
GatewayStep::FmtCheck => run_gateway_fmt_check(gateway_dir),
|
||||
GatewayStep::Lint => run_rebar(gateway_dir, ["lint"]),
|
||||
GatewayStep::Compile => run_rebar(gateway_dir, ["compile"]),
|
||||
GatewayStep::ProdCompile => {
|
||||
run_rebar(gateway_dir, ["as", "prod", "clean", "-a"])?;
|
||||
run_rebar(gateway_dir, ["as", "prod", "compile"])
|
||||
}
|
||||
GatewayStep::Dialyzer => run_rebar(gateway_dir, ["dialyzer"]),
|
||||
GatewayStep::Eqwalizer => run_eqwalizer(gateway_dir, eqwalizer_profile),
|
||||
GatewayStep::Typecheck => run_eqwalizer(gateway_dir, eqwalizer_profile),
|
||||
GatewayStep::Eunit => run_rebar(gateway_dir, ["as", "test", "eunit"]),
|
||||
GatewayStep::Bench => run_rebar(gateway_dir, ["eunit", "--module=guild_member_list_bench"]),
|
||||
GatewayStep::AllChecks => {
|
||||
run_gateway_check_step("Step 1/5: Format check (erlfmt)", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::FmtCheck, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 2/5: Lint (elvis)", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Lint, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 3/5: Compile", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Compile, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 4/5: Type check", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Typecheck, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 5/5: Unit tests (eunit)", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Eunit, eqwalizer_profile)
|
||||
})?;
|
||||
println!("All gateway checks passed.");
|
||||
Ok(())
|
||||
}
|
||||
GatewayStep::Clean => {
|
||||
run_rebar(gateway_dir, ["clean", "--all"])?;
|
||||
let plugins_dir = gateway_dir.join("_build/default/plugins");
|
||||
if plugins_dir.exists() {
|
||||
fs::remove_dir_all(&plugins_dir)
|
||||
.with_context(|| format!("Failed to remove {}", plugins_dir.display()))?;
|
||||
}
|
||||
println!("Cleaned gateway build outputs.");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_gateway_nifs(gateway_dir: &Path) -> Result<()> {
|
||||
let profile =
|
||||
NifBuildProfile::from_env_value(env::var("FLUXER_GATEWAY_NIF_PROFILE").ok().as_deref());
|
||||
let builds = gateway_nif_builds(
|
||||
gateway_dir,
|
||||
profile,
|
||||
env::consts::DLL_PREFIX,
|
||||
env::consts::DLL_EXTENSION,
|
||||
);
|
||||
let priv_dir = gateway_dir.join("priv");
|
||||
fs::create_dir_all(&priv_dir)
|
||||
.with_context(|| format!("Failed to create {}", priv_dir.display()))?;
|
||||
|
||||
for build in builds {
|
||||
let mut cargo_args = build.cargo_args.clone();
|
||||
cargo_args.push(OsString::from("--manifest-path"));
|
||||
cargo_args.push(build.native_dir.join("Cargo.toml").into_os_string());
|
||||
run_command(CommandSpec::new("cargo").args(cargo_args))?;
|
||||
|
||||
ensure!(
|
||||
build.artifact_path.is_file(),
|
||||
"Expected NIF artifact was not produced: {}",
|
||||
build.artifact_path.display()
|
||||
);
|
||||
fs::copy(&build.artifact_path, &build.output_path).with_context(|| {
|
||||
format!(
|
||||
"Failed to copy {} to {}",
|
||||
build.artifact_path.display(),
|
||||
build.output_path.display()
|
||||
)
|
||||
})?;
|
||||
println!(
|
||||
"Installed gateway NIF {} -> {}",
|
||||
build.crate_name,
|
||||
build.output_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_gateway_check_step(label: &str, run: impl FnOnce() -> Result<()>) -> Result<()> {
|
||||
println!("========================================");
|
||||
println!(" {label}");
|
||||
println!("========================================");
|
||||
run()?;
|
||||
println!();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_rebar(
|
||||
gateway_dir: &Path,
|
||||
args: impl IntoIterator<Item = impl Into<OsString>>,
|
||||
) -> Result<()> {
|
||||
run_command(rebar_command(gateway_dir, args))
|
||||
}
|
||||
|
||||
fn run_gateway_fmt_check(gateway_dir: &Path) -> Result<()> {
|
||||
ensure!(
|
||||
command_succeeds(with_asdf_shims(CommandSpec::new("rebar3").arg("--version"))),
|
||||
"rebar3 is required for gateway formatting"
|
||||
);
|
||||
let output = crate::common::capture(rebar_command(gateway_dir, ["fmt", "--check"]))?;
|
||||
if output.status == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let combined = format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
if combined.contains("Command fmt not found")
|
||||
|| combined.to_ascii_lowercase().contains("not found")
|
||||
{
|
||||
println!("rebar3 fmt plugin is not configured; skipping gateway formatting check.");
|
||||
return Ok(());
|
||||
}
|
||||
bail!("gateway formatting failed with exit code {}", output.status)
|
||||
}
|
||||
|
||||
fn run_eqwalizer(gateway_dir: &Path, profile: &str) -> Result<()> {
|
||||
ensure!(
|
||||
!profile.is_empty(),
|
||||
"--eqwalizer-profile requires a non-empty profile name"
|
||||
);
|
||||
ensure!(
|
||||
command_succeeds(with_asdf_shims(CommandSpec::new("elp").arg("version"))),
|
||||
"elp not found in PATH. Install ELP from https://github.com/WhatsApp/erlang-language-platform/releases"
|
||||
);
|
||||
ensure!(
|
||||
command_succeeds(with_asdf_shims(CommandSpec::new("erl").arg("-version"))),
|
||||
"erl not found in PATH"
|
||||
);
|
||||
let erlang_source = find_erlang_source()?;
|
||||
let elp_version = output_text(with_asdf_shims(CommandSpec::new("elp").arg("version")))?;
|
||||
println!("==> Running eqWAlizer with {elp_version}");
|
||||
println!("==> Using Erlang source: {erlang_source}");
|
||||
println!("==> Rebar profile: {profile}");
|
||||
run_command(
|
||||
with_asdf_shims(
|
||||
CommandSpec::new("elp")
|
||||
.args([
|
||||
"eqwalize-all",
|
||||
"--rebar",
|
||||
"--as",
|
||||
profile,
|
||||
"--stats",
|
||||
"--bail-on-error",
|
||||
])
|
||||
.current_dir(gateway_dir),
|
||||
)
|
||||
.env("REBAR_SKIP_PROJECT_PLUGINS", "1"),
|
||||
)
|
||||
}
|
||||
|
||||
fn find_erlang_source() -> Result<String> {
|
||||
output_text(with_asdf_shims(CommandSpec::new("erl").args([
|
||||
"-noshell",
|
||||
"-eval",
|
||||
concat!(
|
||||
"Root = code:root_dir(), ",
|
||||
"Matches = filelib:wildcard(filename:join([Root, \"lib\", \"erts-*\", \"src\", \"erlang.erl\"])), ",
|
||||
"case Matches of ",
|
||||
"[Path | _] -> io:format(\"~s~n\", [Path]), halt(0); ",
|
||||
"[] -> halt(2) ",
|
||||
"end."
|
||||
),
|
||||
])))
|
||||
.context(
|
||||
"Erlang/OTP source files are required for Eqwalizer. On Debian/Ubuntu, install erlang-src",
|
||||
)
|
||||
}
|
||||
|
||||
fn rebar_command(
|
||||
gateway_dir: &Path,
|
||||
args: impl IntoIterator<Item = impl Into<OsString>>,
|
||||
) -> CommandSpec {
|
||||
let args = args.into_iter().map(Into::into).collect::<Vec<_>>();
|
||||
let should_skip_plugins = should_skip_rebar_project_plugins(&args);
|
||||
let mut spec = with_asdf_shims(
|
||||
CommandSpec::new("rebar3")
|
||||
.args(args)
|
||||
.current_dir(gateway_dir),
|
||||
);
|
||||
if should_skip_plugins {
|
||||
spec = spec.env("REBAR_SKIP_PROJECT_PLUGINS", "1");
|
||||
}
|
||||
spec
|
||||
}
|
||||
|
||||
fn should_skip_rebar_project_plugins(args: &[OsString]) -> bool {
|
||||
!args
|
||||
.iter()
|
||||
.any(|arg| matches!(arg.to_string_lossy().as_ref(), "fmt" | "lint" | "plugins"))
|
||||
}
|
||||
|
||||
fn with_asdf_shims(spec: CommandSpec) -> CommandSpec {
|
||||
let Some(shims_path) = asdf_shims_path() else {
|
||||
return spec;
|
||||
};
|
||||
if !shims_path.is_dir() {
|
||||
return spec;
|
||||
}
|
||||
let path = env::var_os("PATH").unwrap_or_default();
|
||||
let mut paths = std::iter::once(shims_path)
|
||||
.chain(env::split_paths(&path))
|
||||
.collect::<Vec<_>>();
|
||||
let joined = env::join_paths(paths.drain(..)).unwrap_or(path);
|
||||
spec.env("PATH", joined)
|
||||
}
|
||||
|
||||
fn asdf_shims_path() -> Option<PathBuf> {
|
||||
if let Some(asdf_data_dir) = env::var_os("ASDF_DATA_DIR") {
|
||||
return Some(PathBuf::from(asdf_data_dir).join("shims"));
|
||||
}
|
||||
env::var_os("HOME")
|
||||
.filter(|home| !home.is_empty())
|
||||
.map(|home| PathBuf::from(home).join(".asdf/shims"))
|
||||
}
|
||||
|
||||
fn gateway_nif_builds(
|
||||
gateway_dir: &Path,
|
||||
profile: NifBuildProfile,
|
||||
dll_prefix: &str,
|
||||
dll_extension: &str,
|
||||
) -> Vec<GatewayNifBuild> {
|
||||
GATEWAY_NIF_CRATES
|
||||
.iter()
|
||||
.map(|crate_name| {
|
||||
gateway_nif_build(gateway_dir, profile, dll_prefix, dll_extension, crate_name)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn gateway_nif_build(
|
||||
gateway_dir: &Path,
|
||||
profile: NifBuildProfile,
|
||||
dll_prefix: &str,
|
||||
dll_extension: &str,
|
||||
crate_name: &str,
|
||||
) -> GatewayNifBuild {
|
||||
let native_dir = gateway_dir.join("native").join(crate_name);
|
||||
GatewayNifBuild {
|
||||
crate_name: crate_name.to_string(),
|
||||
cargo_args: profile.cargo_args(),
|
||||
artifact_path: native_dir
|
||||
.join("target")
|
||||
.join(profile.target_dir_name())
|
||||
.join(format!("{dll_prefix}{crate_name}.{dll_extension}")),
|
||||
output_path: gateway_dir.join("priv").join(format!("{crate_name}.so")),
|
||||
native_dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_gateway_dir() -> Result<PathBuf> {
|
||||
let cwd = env::current_dir().context("Failed to resolve current directory")?;
|
||||
if cwd.join("rebar.config").is_file()
|
||||
&& cwd.file_name().and_then(|value| value.to_str()) == Some("fluxer_gateway")
|
||||
{
|
||||
return Ok(cwd);
|
||||
}
|
||||
if cwd.join("fluxer_gateway/rebar.config").is_file() {
|
||||
return Ok(cwd.join("fluxer_gateway"));
|
||||
}
|
||||
Err(anyhow!(
|
||||
"Could not resolve fluxer_gateway directory from {}",
|
||||
cwd.display()
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn nif_profile_defaults_to_release_and_uses_debug_for_other_values() {
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(None),
|
||||
NifBuildProfile::Release
|
||||
);
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(Some("release")),
|
||||
NifBuildProfile::Release
|
||||
);
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(Some("debug")),
|
||||
NifBuildProfile::Debug
|
||||
);
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(Some("dev")),
|
||||
NifBuildProfile::Debug
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_nif_build_plan_matches_legacy_artifact_layout() {
|
||||
let gateway_dir = Path::new("/repo/fluxer_gateway");
|
||||
let builds = gateway_nif_builds(gateway_dir, NifBuildProfile::Release, "lib", "so");
|
||||
|
||||
assert_eq!(builds.len(), 2);
|
||||
assert_eq!(
|
||||
builds[0],
|
||||
GatewayNifBuild {
|
||||
crate_name: "push_markdown_plaintext_nif".to_string(),
|
||||
native_dir: PathBuf::from(
|
||||
"/repo/fluxer_gateway/native/push_markdown_plaintext_nif"
|
||||
),
|
||||
cargo_args: vec![OsString::from("build"), OsString::from("--release")],
|
||||
artifact_path: PathBuf::from(
|
||||
"/repo/fluxer_gateway/native/push_markdown_plaintext_nif/target/release/libpush_markdown_plaintext_nif.so"
|
||||
),
|
||||
output_path: PathBuf::from(
|
||||
"/repo/fluxer_gateway/priv/push_markdown_plaintext_nif.so"
|
||||
),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_build_plan_omits_release_arg_and_uses_debug_target_dir() {
|
||||
let build = gateway_nif_build(
|
||||
Path::new("/repo/fluxer_gateway"),
|
||||
NifBuildProfile::Debug,
|
||||
"lib",
|
||||
"dylib",
|
||||
"guild_member_list_oset_nif",
|
||||
);
|
||||
|
||||
assert_eq!(build.cargo_args, vec![OsString::from("build")]);
|
||||
assert_eq!(
|
||||
build.artifact_path,
|
||||
PathBuf::from(
|
||||
"/repo/fluxer_gateway/native/guild_member_list_oset_nif/target/debug/libguild_member_list_oset_nif.dylib"
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
build.output_path,
|
||||
PathBuf::from("/repo/fluxer_gateway/priv/guild_member_list_oset_nif.so")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebar_project_plugins_are_skipped_except_for_plugin_commands() {
|
||||
assert!(should_skip_rebar_project_plugins(&[OsString::from(
|
||||
"compile"
|
||||
)]));
|
||||
assert!(should_skip_rebar_project_plugins(&[
|
||||
OsString::from("as"),
|
||||
OsString::from("test"),
|
||||
OsString::from("eunit"),
|
||||
]));
|
||||
assert!(!should_skip_rebar_project_plugins(&[OsString::from("fmt")]));
|
||||
assert!(!should_skip_rebar_project_plugins(&[OsString::from(
|
||||
"lint"
|
||||
)]));
|
||||
assert!(!should_skip_rebar_project_plugins(&[OsString::from(
|
||||
"plugins"
|
||||
)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebar_command_runs_in_gateway_dir_and_sets_skip_env_for_compile() {
|
||||
let command = rebar_command(Path::new("/repo/fluxer_gateway"), ["compile"]);
|
||||
|
||||
assert_eq!(command.program, OsString::from("rebar3"));
|
||||
assert_eq!(command.args, vec![OsString::from("compile")]);
|
||||
assert_eq!(command.cwd, Some(PathBuf::from("/repo/fluxer_gateway")));
|
||||
assert!(command.env.contains(&(
|
||||
OsString::from("REBAR_SKIP_PROJECT_PLUGINS"),
|
||||
OsString::from("1")
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebar_command_keeps_project_plugins_for_fmt() {
|
||||
let command = rebar_command(Path::new("/repo/fluxer_gateway"), ["fmt", "--check"]);
|
||||
|
||||
assert_eq!(
|
||||
command.args,
|
||||
vec![OsString::from("fmt"), OsString::from("--check")]
|
||||
);
|
||||
assert!(
|
||||
!command
|
||||
.env
|
||||
.iter()
|
||||
.any(|(key, _)| key == &OsString::from("REBAR_SKIP_PROJECT_PLUGINS"))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
mod app_dev_server;
|
||||
mod app_proxy;
|
||||
mod app_wasm;
|
||||
mod calver;
|
||||
mod ci_workflow;
|
||||
mod common;
|
||||
mod desktop;
|
||||
mod desktop_native;
|
||||
mod gateway;
|
||||
mod schema;
|
||||
mod static_bucket;
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "fluxer-ci")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum Command {
|
||||
AppDevServer(app_dev_server::AppDevServerArgs),
|
||||
BuildAppWasm(app_wasm::BuildAppWasmArgs),
|
||||
BuildAppProxy(app_proxy::BuildAppProxyArgs),
|
||||
BuildMarkdownParserWasm(app_wasm::BuildMarkdownParserWasmArgs),
|
||||
BuildDesktop(desktop::BuildDesktopArgs),
|
||||
BuildDesktopNativeAddon(desktop_native::BuildDesktopNativeAddonArgs),
|
||||
BuildGatewayNifs(gateway::BuildGatewayNifsArgs),
|
||||
Ci(ci_workflow::CiArgs),
|
||||
CiScripts(ci_workflow::CiScriptsArgs),
|
||||
CleanSchemaGeneratedFiles(schema::CleanSchemaGeneratedFilesArgs),
|
||||
Gateway(gateway::GatewayArgs),
|
||||
RepairStaticAssetMetadata(static_bucket::RepairStaticAssetMetadataArgs),
|
||||
ResolveCalver(calver::ResolveCalverArgs),
|
||||
SyncStaticBucket(static_bucket::SyncStaticBucketArgs),
|
||||
TestWebrtcSenderRust(desktop_native::TestWebrtcSenderRustArgs),
|
||||
}
|
||||
|
||||
pub async fn run() -> Result<()> {
|
||||
match Cli::parse().command {
|
||||
Command::AppDevServer(args) => app_dev_server::run(args).await,
|
||||
Command::BuildAppWasm(args) => app_wasm::run_build_app_wasm(args),
|
||||
Command::BuildAppProxy(args) => app_proxy::run(args).await,
|
||||
Command::BuildMarkdownParserWasm(args) => app_wasm::run_build_markdown_parser_wasm(args),
|
||||
Command::BuildDesktop(args) => desktop::run(args).await,
|
||||
Command::BuildDesktopNativeAddon(args) => {
|
||||
desktop_native::run_build_desktop_native_addon(args)
|
||||
}
|
||||
Command::BuildGatewayNifs(args) => gateway::run_build_gateway_nifs(args),
|
||||
Command::Ci(args) => ci_workflow::run_ci(args).await,
|
||||
Command::CiScripts(args) => ci_workflow::run_ci_scripts(args).await,
|
||||
Command::CleanSchemaGeneratedFiles(args) => schema::run_clean_generated_files(args),
|
||||
Command::Gateway(args) => gateway::run_gateway(args),
|
||||
Command::RepairStaticAssetMetadata(args) => {
|
||||
static_bucket::repair_asset_metadata(args).await
|
||||
}
|
||||
Command::ResolveCalver(args) => calver::run(args),
|
||||
Command::SyncStaticBucket(args) => static_bucket::run(args).await,
|
||||
Command::TestWebrtcSenderRust(args) => desktop_native::run_test_webrtc_sender_rust(args),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
if let Err(error) = fluxer_ci::run().await {
|
||||
eprintln!("{error:?}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::collect_files;
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Args;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct CleanSchemaGeneratedFilesArgs {
|
||||
#[arg(long, default_value = "packages/schema/src/gen")]
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
pub fn run_clean_generated_files(args: CleanSchemaGeneratedFilesArgs) -> Result<()> {
|
||||
clean_generated_files(&args.root)
|
||||
}
|
||||
|
||||
fn clean_generated_files(root: &Path) -> Result<()> {
|
||||
for file in collect_files(root)?
|
||||
.into_iter()
|
||||
.filter(|path| path.extension().and_then(|value| value.to_str()) == Some("ts"))
|
||||
{
|
||||
let source = fs::read_to_string(&file)
|
||||
.with_context(|| format!("Failed to read {}", file.display()))?;
|
||||
let Some(import_index) = find_import_start(&source) else {
|
||||
continue;
|
||||
};
|
||||
let content = format!(
|
||||
"{}\n",
|
||||
collapse_extra_blank_lines(source[import_index..].trim_end())
|
||||
);
|
||||
if content != source {
|
||||
fs::write(&file, content)
|
||||
.with_context(|| format!("Failed to write {}", file.display()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn find_import_start(source: &str) -> Option<usize> {
|
||||
let mut offset = 0usize;
|
||||
for line in source.split_inclusive('\n') {
|
||||
if line.starts_with("import ") {
|
||||
return Some(offset);
|
||||
}
|
||||
offset += line.len();
|
||||
}
|
||||
if source[offset..].starts_with("import ") {
|
||||
return Some(offset);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn collapse_extra_blank_lines(source: &str) -> String {
|
||||
let mut output = String::with_capacity(source.len());
|
||||
let mut consecutive_newlines = 0usize;
|
||||
for ch in source.chars() {
|
||||
if ch == '\n' {
|
||||
consecutive_newlines += 1;
|
||||
if consecutive_newlines <= 2 {
|
||||
output.push(ch);
|
||||
}
|
||||
} else {
|
||||
consecutive_newlines = 0;
|
||||
output.push(ch);
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write_file(path: &Path, contents: &str) {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_import_start_only_matches_line_starts() {
|
||||
assert_eq!(
|
||||
find_import_start("// import nope\nimport {x} from 'x';\n"),
|
||||
Some(15)
|
||||
);
|
||||
assert_eq!(find_import_start("const value = 'import nope';\n"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collapse_extra_blank_lines_keeps_at_most_one_blank_line() {
|
||||
assert_eq!(collapse_extra_blank_lines("a\n\n\n\nb\n\nc"), "a\n\nb\n\nc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_generated_files_removes_prelude_and_compacts_blank_lines() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path();
|
||||
let file = root.join("generated.ts");
|
||||
write_file(
|
||||
&file,
|
||||
"// header\n\n\nimport {x} from 'x';\n\n\n\nexport const y = x;\n\n",
|
||||
);
|
||||
write_file(&root.join("keep.txt"), "// header\n");
|
||||
|
||||
clean_generated_files(root).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
fs::read_to_string(file).unwrap(),
|
||||
"import {x} from 'x';\n\nexport const y = x;\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{
|
||||
S3UploadPlanItem, collect_files, delete_s3_objects, list_s3_keys, path_to_s3_key,
|
||||
replace_s3_object_metadata, s3_client, s3_content_type_for_key, upload_s3_plan_sync,
|
||||
};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use clap::Args;
|
||||
use std::collections::BTreeMap;
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
const DEFAULT_SOURCE: &str = "fluxer_static";
|
||||
const DEFAULT_STATIC_BUCKET: &str = "fluxer-static";
|
||||
const DEFAULT_S3_ENDPOINT: &str = "https://ewr1.vultrobjects.com";
|
||||
const DEFAULT_ASSET_PREFIX: &str = "assets/";
|
||||
const IMMUTABLE_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable";
|
||||
const DEFAULT_REPAIR_CONCURRENCY: usize = 8;
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct SyncStaticBucketArgs {
|
||||
#[arg(long, default_value = DEFAULT_SOURCE)]
|
||||
source: PathBuf,
|
||||
#[arg(long, default_value = DEFAULT_STATIC_BUCKET)]
|
||||
bucket: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct RepairStaticAssetMetadataArgs {
|
||||
#[arg(long, default_value = DEFAULT_STATIC_BUCKET)]
|
||||
bucket: String,
|
||||
#[arg(long, default_value = DEFAULT_ASSET_PREFIX)]
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
pub async fn run(args: SyncStaticBucketArgs) -> Result<()> {
|
||||
ensure!(
|
||||
args.source.is_dir(),
|
||||
"Static source directory is missing: {}",
|
||||
args.source.display()
|
||||
);
|
||||
|
||||
let upload_plan = static_upload_plan(&args.source)?;
|
||||
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
|
||||
|
||||
let plan = upload_plan
|
||||
.into_iter()
|
||||
.map(|(key, path)| S3UploadPlanItem::new(path, key).with_detected_content_type())
|
||||
.collect::<Vec<_>>();
|
||||
let stats = upload_s3_plan_sync(&client, &args.bucket, plan).await?;
|
||||
|
||||
println!(
|
||||
"Static bucket sync complete: uploaded {} file(s), skipped existing {}",
|
||||
stats.uploaded, stats.skipped_existing
|
||||
);
|
||||
|
||||
let remote_keys = list_s3_keys(&client, &args.bucket, "").await?;
|
||||
let markdown_keys = remote_keys
|
||||
.into_iter()
|
||||
.filter(|key| key.to_ascii_lowercase().ends_with(".md"))
|
||||
.collect::<Vec<_>>();
|
||||
let removed = delete_s3_objects(&client, &args.bucket, &markdown_keys).await?;
|
||||
println!("Static bucket sync removed {removed} stray .md object(s)");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn repair_asset_metadata(args: RepairStaticAssetMetadataArgs) -> Result<()> {
|
||||
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
|
||||
let keys = list_s3_keys(&client, &args.bucket, &args.prefix).await?;
|
||||
let mut skipped = 0_usize;
|
||||
let mut tasks = JoinSet::new();
|
||||
let semaphore = Arc::new(Semaphore::new(static_asset_repair_concurrency()));
|
||||
|
||||
for key in keys {
|
||||
let Some(content_type) = s3_content_type_for_key(&key) else {
|
||||
skipped += 1;
|
||||
continue;
|
||||
};
|
||||
let permit = semaphore
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.context("S3 repair semaphore closed")?;
|
||||
let client = client.clone();
|
||||
let bucket = args.bucket.clone();
|
||||
tasks.spawn(async move {
|
||||
let _permit = permit;
|
||||
replace_s3_object_metadata(
|
||||
&client,
|
||||
&bucket,
|
||||
&key,
|
||||
Some(content_type),
|
||||
Some(IMMUTABLE_ASSET_CACHE_CONTROL),
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
});
|
||||
}
|
||||
|
||||
let mut repaired = 0_usize;
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
result.context("S3 metadata repair task failed")??;
|
||||
repaired += 1;
|
||||
}
|
||||
|
||||
println!(
|
||||
"Static asset metadata repair complete: repaired {repaired} file(s), skipped {skipped}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn static_asset_repair_concurrency() -> usize {
|
||||
env::var("S3_WRITE_CONCURRENCY")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(DEFAULT_REPAIR_CONCURRENCY)
|
||||
}
|
||||
|
||||
fn static_upload_plan(source: &Path) -> Result<BTreeMap<String, PathBuf>> {
|
||||
let mut plan = BTreeMap::new();
|
||||
for file in collect_files(source)? {
|
||||
let relative = file
|
||||
.strip_prefix(source)
|
||||
.with_context(|| format!("Failed to relativize {}", file.display()))?;
|
||||
if !should_sync_static_path(relative) {
|
||||
continue;
|
||||
}
|
||||
let key = path_to_s3_key(relative);
|
||||
if !key.is_empty() {
|
||||
plan.insert(key, file);
|
||||
}
|
||||
}
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
fn should_sync_static_path(relative: &Path) -> bool {
|
||||
if relative
|
||||
.extension()
|
||||
.and_then(|ext| ext.to_str())
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(first) = relative.components().next() else {
|
||||
return false;
|
||||
};
|
||||
match first {
|
||||
std::path::Component::Normal(value) => {
|
||||
let value = value.to_string_lossy();
|
||||
value != ".github" && value != "assets"
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
fn write_file(path: &Path, contents: &str) {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_plan_excludes_github_and_assets_roots() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let source = temp.path();
|
||||
write_file(&source.join("index.html"), "html");
|
||||
write_file(&source.join("docs/install.html"), "docs");
|
||||
write_file(&source.join(".github/workflows/ignored.yaml"), "workflow");
|
||||
write_file(&source.join("assets/app.js"), "asset");
|
||||
|
||||
let keys = static_upload_plan(source)
|
||||
.unwrap()
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(keys, vec!["docs/install.html", "index.html"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_plan_is_deterministic_and_keeps_non_root_assets_paths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let source = temp.path();
|
||||
write_file(&source.join("z.html"), "z");
|
||||
write_file(&source.join("docs/assets/keep.js"), "keep");
|
||||
write_file(&source.join("a.html"), "a");
|
||||
|
||||
let keys = static_upload_plan(source)
|
||||
.unwrap()
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(keys, vec!["a.html", "docs/assets/keep.js", "z.html"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn static_key_filter_matches_workflow_excludes() {
|
||||
assert!(should_sync_static_path(Path::new("index.html")));
|
||||
assert!(should_sync_static_path(
|
||||
Path::new("docs").join("install.html").as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new("assets").join("app.js").as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new(".github")
|
||||
.join("workflows")
|
||||
.join("sync.yaml")
|
||||
.as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(Path::new("")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn static_key_filter_excludes_markdown() {
|
||||
assert!(!should_sync_static_path(Path::new(
|
||||
"THIRD_PARTY_LICENSES.md"
|
||||
)));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new("fonts").join("NOTICE.md").as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new("emoji").join("README.MD").as_path()
|
||||
));
|
||||
assert!(should_sync_static_path(Path::new("index.html")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_plan_excludes_markdown_files() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let source = temp.path();
|
||||
write_file(&source.join("index.html"), "html");
|
||||
write_file(&source.join("THIRD_PARTY_LICENSES.md"), "licenses");
|
||||
write_file(&source.join("fonts/NOTICE.md"), "notice");
|
||||
|
||||
let keys = static_upload_plan(source)
|
||||
.unwrap()
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(keys, vec!["index.html"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repair_metadata_args_default_to_app_assets_prefix() {
|
||||
let args = RepairStaticAssetMetadataArgs {
|
||||
bucket: DEFAULT_STATIC_BUCKET.to_string(),
|
||||
prefix: DEFAULT_ASSET_PREFIX.to_string(),
|
||||
};
|
||||
assert_eq!(args.bucket, "fluxer-static");
|
||||
assert_eq!(args.prefix, "assets/");
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
export interface RgbaTransformResult {
|
||||
rgba: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface DecodedFrame {
|
||||
rgba: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
delayMs: number;
|
||||
}
|
||||
|
||||
export interface EncodedApngFrame {
|
||||
compressed: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
delayMs: number;
|
||||
}
|
||||
|
||||
export interface EncodedGifChunk {
|
||||
data: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export function assemble_apng_frames(frames: Array<EncodedApngFrame>): Uint8Array;
|
||||
export function assemble_gif_frame_chunks(chunks: Array<EncodedGifChunk>): Uint8Array;
|
||||
export function crop_and_rotate_apng(
|
||||
input: Uint8Array,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): Uint8Array;
|
||||
export function crop_and_rotate_gif(
|
||||
input: Uint8Array,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): Uint8Array;
|
||||
export function crop_and_rotate_image(
|
||||
input: Uint8Array,
|
||||
format_hint: string,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): Uint8Array;
|
||||
export function crop_rotate_rgba(
|
||||
input: Uint8Array,
|
||||
src_width: number,
|
||||
src_height: number,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): RgbaTransformResult;
|
||||
export function decode_apng_frames(input: Uint8Array): Array<DecodedFrame>;
|
||||
export function decode_gif_frames(input: Uint8Array): Array<DecodedFrame>;
|
||||
export function encode_apng_frame_payload(frame: DecodedFrame): EncodedApngFrame;
|
||||
export function encode_apng_frames(frames: Array<DecodedFrame>): Uint8Array;
|
||||
export function encode_gif_frame_chunk(frame: DecodedFrame, first?: boolean): EncodedGifChunk;
|
||||
export function encode_gif_frames(frames: Array<DecodedFrame>): Uint8Array;
|
||||
@@ -0,0 +1,757 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {encode as encodePng} from 'fast-png';
|
||||
import {zlibSync} from 'fflate';
|
||||
import {applyPalette, GIFEncoder, quantize} from 'gifenc';
|
||||
import {decompressFrames, parseGIF} from 'gifuct-js';
|
||||
import {decode as decodeJpeg, encode as encodeJpeg} from 'jpeg-js';
|
||||
import UPNG from 'upng-js';
|
||||
|
||||
const textEncoder = new TextEncoder();
|
||||
const NULL_U32 = 0xffffffff;
|
||||
const RGBA_RESULT_HEADER_BYTES = 8;
|
||||
const MAX_ANIMATION_PIXELS = 200_000_000;
|
||||
const MAX_STATIC_DECODE_MEMORY_MB = 1024;
|
||||
|
||||
function inputBytes(input) {
|
||||
if (input == null) return new Uint8Array();
|
||||
return input instanceof Uint8Array ? input : new Uint8Array(input);
|
||||
}
|
||||
|
||||
function arrayBufferFor(bytes) {
|
||||
if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) return bytes.buffer;
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||
}
|
||||
|
||||
function nonNegativeU32(value) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number <= 0) return 0;
|
||||
return Math.min(NULL_U32, Math.floor(number));
|
||||
}
|
||||
|
||||
function cropCoordU32(value) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number <= 0) return 0;
|
||||
return Math.min(NULL_U32, Math.floor(number));
|
||||
}
|
||||
|
||||
function optionalDimension(value) {
|
||||
const dimension = nonNegativeU32(value);
|
||||
return dimension > 0 ? dimension : null;
|
||||
}
|
||||
|
||||
function effectiveRotation(rotationDeg) {
|
||||
const rotation = ((Math.floor(Number(rotationDeg) || 0) % 360) + 360) % 360;
|
||||
return rotation === 90 || rotation === 180 || rotation === 270 ? rotation : 0;
|
||||
}
|
||||
|
||||
function normalizedFormat(value) {
|
||||
const format = String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (format === 'jpg') return 'jpeg';
|
||||
if (format === 'apng') return 'png';
|
||||
if (format === 'animated_webp') return 'webp';
|
||||
return format;
|
||||
}
|
||||
|
||||
function sniffImageFormat(input) {
|
||||
const bytes = inputBytes(input);
|
||||
if (
|
||||
bytes.length >= 8 &&
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47 &&
|
||||
bytes[4] === 0x0d &&
|
||||
bytes[5] === 0x0a &&
|
||||
bytes[6] === 0x1a &&
|
||||
bytes[7] === 0x0a
|
||||
) {
|
||||
return 'png';
|
||||
}
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'jpeg';
|
||||
if (
|
||||
bytes.length >= 6 &&
|
||||
bytes[0] === 0x47 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x38 &&
|
||||
(bytes[4] === 0x37 || bytes[4] === 0x39) &&
|
||||
bytes[5] === 0x61
|
||||
) {
|
||||
return 'gif';
|
||||
}
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
bytes[0] === 0x52 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x46 &&
|
||||
bytes[8] === 0x57 &&
|
||||
bytes[9] === 0x45 &&
|
||||
bytes[10] === 0x42 &&
|
||||
bytes[11] === 0x50
|
||||
) {
|
||||
return 'webp';
|
||||
}
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
bytes[4] === 0x66 &&
|
||||
bytes[5] === 0x74 &&
|
||||
bytes[6] === 0x79 &&
|
||||
bytes[7] === 0x70 &&
|
||||
bytes[8] === 0x61 &&
|
||||
bytes[9] === 0x76 &&
|
||||
bytes[10] === 0x69 &&
|
||||
(bytes[11] === 0x66 || bytes[11] === 0x73)
|
||||
) {
|
||||
return 'avif';
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function readU32FromBytes(bytes, offset) {
|
||||
return new DataView(bytes.buffer, bytes.byteOffset + offset, 4).getUint32(0, true);
|
||||
}
|
||||
|
||||
function readU16LE(bytes, offset) {
|
||||
return bytes[offset] | (bytes[offset + 1] << 8);
|
||||
}
|
||||
|
||||
function readU16BE(bytes, offset) {
|
||||
return (bytes[offset] << 8) | bytes[offset + 1];
|
||||
}
|
||||
|
||||
function readU32BE(bytes, offset) {
|
||||
return (bytes[offset] * 0x1000000 + ((bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3])) >>> 0;
|
||||
}
|
||||
|
||||
function parseRgbaTransformResult(bytes) {
|
||||
if (bytes.byteLength < RGBA_RESULT_HEADER_BYTES) throw new Error('libfluxcore returned a truncated RGBA result');
|
||||
const width = readU32FromBytes(bytes, 0);
|
||||
const height = readU32FromBytes(bytes, 4);
|
||||
const expected = RGBA_RESULT_HEADER_BYTES + width * height * 4;
|
||||
if (bytes.byteLength !== expected) throw new Error('libfluxcore returned an invalid RGBA result length');
|
||||
return {rgba: bytes.subarray(RGBA_RESULT_HEADER_BYTES), width, height};
|
||||
}
|
||||
|
||||
function pngDimensions(bytes) {
|
||||
if (bytes.length < 24 || sniffImageFormat(bytes) !== 'png') return null;
|
||||
if (bytes[12] !== 0x49 || bytes[13] !== 0x48 || bytes[14] !== 0x44 || bytes[15] !== 0x52) return null;
|
||||
return {width: readU32BE(bytes, 16), height: readU32BE(bytes, 20)};
|
||||
}
|
||||
|
||||
function gifDimensions(bytes) {
|
||||
if (bytes.length < 10 || sniffImageFormat(bytes) !== 'gif') return null;
|
||||
return {width: readU16LE(bytes, 6), height: readU16LE(bytes, 8)};
|
||||
}
|
||||
|
||||
function jpegDimensions(bytes) {
|
||||
if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null;
|
||||
let offset = 2;
|
||||
while (offset + 4 <= bytes.length) {
|
||||
while (offset < bytes.length && bytes[offset] === 0xff) offset += 1;
|
||||
if (offset >= bytes.length) return null;
|
||||
const marker = bytes[offset];
|
||||
offset += 1;
|
||||
if (marker === 0xd9 || marker === 0xda) return null;
|
||||
if (offset + 2 > bytes.length) return null;
|
||||
const length = readU16BE(bytes, offset);
|
||||
if (length < 2 || offset + length > bytes.length) return null;
|
||||
if (
|
||||
marker === 0xc0 ||
|
||||
marker === 0xc1 ||
|
||||
marker === 0xc2 ||
|
||||
marker === 0xc3 ||
|
||||
marker === 0xc5 ||
|
||||
marker === 0xc6 ||
|
||||
marker === 0xc7 ||
|
||||
marker === 0xc9 ||
|
||||
marker === 0xca ||
|
||||
marker === 0xcb ||
|
||||
marker === 0xcd ||
|
||||
marker === 0xce ||
|
||||
marker === 0xcf
|
||||
) {
|
||||
if (length < 7) return null;
|
||||
return {height: readU16BE(bytes, offset + 3), width: readU16BE(bytes, offset + 5)};
|
||||
}
|
||||
offset += length;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function imageDimensions(bytes, format) {
|
||||
switch (format) {
|
||||
case 'png':
|
||||
return pngDimensions(bytes);
|
||||
case 'gif':
|
||||
return gifDimensions(bytes);
|
||||
case 'jpeg':
|
||||
return jpegDimensions(bytes);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isNoopTransform(imageWidth, imageHeight, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
const cropX = Math.min(cropCoordU32(x), imageWidth);
|
||||
const cropY = Math.min(cropCoordU32(y), imageHeight);
|
||||
const cropW = Math.min(nonNegativeU32(width), imageWidth - cropX);
|
||||
const cropH = Math.min(nonNegativeU32(height), imageHeight - cropY);
|
||||
const targetW = optionalDimension(resizeWidth) ?? imageWidth;
|
||||
const targetH = optionalDimension(resizeHeight) ?? imageHeight;
|
||||
return (
|
||||
cropX === 0 &&
|
||||
cropY === 0 &&
|
||||
cropW === imageWidth &&
|
||||
cropH === imageHeight &&
|
||||
effectiveRotation(rotationDeg) === 0 &&
|
||||
targetW === imageWidth &&
|
||||
targetH === imageHeight
|
||||
);
|
||||
}
|
||||
|
||||
export function crop_rotate_rgba(
|
||||
input,
|
||||
src_width,
|
||||
src_height,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotation_deg,
|
||||
resize_width,
|
||||
resize_height,
|
||||
) {
|
||||
const sourceWidth = nonNegativeU32(src_width);
|
||||
const sourceHeight = nonNegativeU32(src_height);
|
||||
return parseRgbaTransformResult(
|
||||
// biome-ignore lint/correctness/noUndeclaredVariables: Injected by the generated wasm wrapper at build time.
|
||||
crop_rotate_rgba_raw(
|
||||
inputBytes(input),
|
||||
sourceWidth,
|
||||
sourceHeight,
|
||||
cropCoordU32(x),
|
||||
cropCoordU32(y),
|
||||
nonNegativeU32(width),
|
||||
nonNegativeU32(height),
|
||||
effectiveRotation(rotation_deg),
|
||||
optionalDimension(resize_width),
|
||||
optionalDimension(resize_height),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function transformFrame(frame, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
const transformed = crop_rotate_rgba(
|
||||
frame.rgba,
|
||||
frame.width,
|
||||
frame.height,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotationDeg,
|
||||
resizeWidth,
|
||||
resizeHeight,
|
||||
);
|
||||
return {rgba: transformed.rgba, width: transformed.width, height: transformed.height, delayMs: frame.delayMs};
|
||||
}
|
||||
|
||||
function transformFrames(frames, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
let totalPixels = 0;
|
||||
const transformed = frames.map((frame) => {
|
||||
const out = transformFrame(frame, x, y, width, height, rotationDeg, resizeWidth, resizeHeight);
|
||||
totalPixels += out.width * out.height;
|
||||
if (totalPixels > MAX_ANIMATION_PIXELS) {
|
||||
throw new Error('Animated image is too large to crop. Try reducing its dimensions or number of frames.');
|
||||
}
|
||||
return out;
|
||||
});
|
||||
return transformed;
|
||||
}
|
||||
|
||||
function decodePngFrames(input) {
|
||||
const bytes = inputBytes(input);
|
||||
const decoded = UPNG.decode(arrayBufferFor(bytes));
|
||||
const rgbaFrames = UPNG.toRGBA8(decoded);
|
||||
if (!rgbaFrames.length) throw new Error('PNG has no frames');
|
||||
return rgbaFrames.map((frame, index) => {
|
||||
const delayMs = decoded.frames?.[index]?.delay ?? 0;
|
||||
return {rgba: new Uint8Array(frame), width: decoded.width, height: decoded.height, delayMs};
|
||||
});
|
||||
}
|
||||
|
||||
function decodeJpegFrame(input) {
|
||||
const decoded = decodeJpeg(inputBytes(input), {
|
||||
useTArray: true,
|
||||
formatAsRGBA: true,
|
||||
maxMemoryUsageInMB: MAX_STATIC_DECODE_MEMORY_MB,
|
||||
});
|
||||
return {rgba: new Uint8Array(decoded.data), width: decoded.width, height: decoded.height, delayMs: 0};
|
||||
}
|
||||
|
||||
function drawGifPatch(canvas, canvasWidth, frame) {
|
||||
const dims = frame.dims;
|
||||
const patch = frame.patch;
|
||||
for (let row = 0; row < dims.height; row += 1) {
|
||||
const canvasY = dims.top + row;
|
||||
if (canvasY < 0) continue;
|
||||
const canvasOffset = (canvasY * canvasWidth + dims.left) * 4;
|
||||
const patchOffset = row * dims.width * 4;
|
||||
if (canvasOffset < 0 || canvasOffset >= canvas.length) continue;
|
||||
for (let col = 0; col < dims.width; col += 1) {
|
||||
const source = patchOffset + col * 4;
|
||||
const target = canvasOffset + col * 4;
|
||||
if (target < 0 || target + 4 > canvas.length || source + 4 > patch.length) continue;
|
||||
if (patch[source + 3] === 0) continue;
|
||||
canvas[target] = patch[source];
|
||||
canvas[target + 1] = patch[source + 1];
|
||||
canvas[target + 2] = patch[source + 2];
|
||||
canvas[target + 3] = patch[source + 3];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function clearRectRgba(canvas, canvasWidth, x, y, width, height) {
|
||||
for (let row = 0; row < height; row += 1) {
|
||||
const start = ((y + row) * canvasWidth + x) * 4;
|
||||
const end = start + width * 4;
|
||||
if (start >= 0 && end <= canvas.length) canvas.fill(0, start, end);
|
||||
}
|
||||
}
|
||||
|
||||
function collectGifFrames(input, transformOptions) {
|
||||
const bytes = inputBytes(input);
|
||||
const parsed = parseGIF(arrayBufferFor(bytes));
|
||||
const screenWidth = parsed.lsd.width;
|
||||
const screenHeight = parsed.lsd.height;
|
||||
const decodedFrames = decompressFrames(parsed, true);
|
||||
if (!decodedFrames.length) throw new Error('GIF has no frames');
|
||||
const canvas = new Uint8Array(screenWidth * screenHeight * 4);
|
||||
let previousCanvas = null;
|
||||
const frames = [];
|
||||
for (const frame of decodedFrames) {
|
||||
if (frame.disposalType === 3) previousCanvas = canvas.slice();
|
||||
drawGifPatch(canvas, screenWidth, frame);
|
||||
const sourceFrame = {
|
||||
rgba: transformOptions ? canvas : canvas.slice(),
|
||||
width: screenWidth,
|
||||
height: screenHeight,
|
||||
delayMs: frame.delay || 100,
|
||||
};
|
||||
frames.push(
|
||||
transformOptions
|
||||
? transformFrame(
|
||||
sourceFrame,
|
||||
transformOptions.x,
|
||||
transformOptions.y,
|
||||
transformOptions.width,
|
||||
transformOptions.height,
|
||||
transformOptions.rotationDeg,
|
||||
transformOptions.resizeWidth,
|
||||
transformOptions.resizeHeight,
|
||||
)
|
||||
: sourceFrame,
|
||||
);
|
||||
if (frame.disposalType === 2) {
|
||||
clearRectRgba(canvas, screenWidth, frame.dims.left, frame.dims.top, frame.dims.width, frame.dims.height);
|
||||
} else if (frame.disposalType === 3 && previousCanvas) {
|
||||
canvas.set(previousCanvas);
|
||||
previousCanvas = null;
|
||||
}
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
function decodeGifFrames(input) {
|
||||
return collectGifFrames(input, null);
|
||||
}
|
||||
|
||||
function transformGifFrames(input, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
return collectGifFrames(input, {x, y, width, height, rotationDeg, resizeWidth, resizeHeight});
|
||||
}
|
||||
|
||||
function exactGifFrameData(rgba) {
|
||||
const palette = [];
|
||||
const colorToIndex = new Map();
|
||||
const index = new Uint8Array(rgba.length / 4);
|
||||
let transparentIndex = -1;
|
||||
for (let offset = 0, pixel = 0; offset < rgba.length; offset += 4, pixel += 1) {
|
||||
if (rgba[offset + 3] === 0) {
|
||||
if (transparentIndex === -1) {
|
||||
if (palette.length >= 256) return null;
|
||||
transparentIndex = palette.length;
|
||||
palette.push([0, 0, 0]);
|
||||
}
|
||||
index[pixel] = transparentIndex;
|
||||
continue;
|
||||
}
|
||||
const key = `${rgba[offset]},${rgba[offset + 1]},${rgba[offset + 2]}`;
|
||||
let paletteIndex = colorToIndex.get(key);
|
||||
if (paletteIndex == null) {
|
||||
if (palette.length >= 256) return null;
|
||||
paletteIndex = palette.length;
|
||||
colorToIndex.set(key, paletteIndex);
|
||||
palette.push([rgba[offset], rgba[offset + 1], rgba[offset + 2]]);
|
||||
}
|
||||
index[pixel] = paletteIndex;
|
||||
}
|
||||
if (palette.length === 0) palette.push([0, 0, 0]);
|
||||
return {index, palette, transparentIndex};
|
||||
}
|
||||
|
||||
function quantizedGifFrameData(rgba) {
|
||||
const input = rgba.byteOffset === 0 && rgba.byteLength === rgba.buffer.byteLength ? rgba : new Uint8Array(rgba);
|
||||
const palette = quantize(input, 256, {format: 'rgba4444', oneBitAlpha: true});
|
||||
const index = applyPalette(input, palette, 'rgba4444');
|
||||
const transparentIndex = palette.findIndex((color) => color.length >= 4 && color[3] === 0);
|
||||
const rgbPalette = palette.map((color) => [color[0], color[1], color[2]]);
|
||||
return {index, palette: rgbPalette, transparentIndex};
|
||||
}
|
||||
|
||||
function writeGifFrame(gif, frame, width, height, first) {
|
||||
if (frame.width !== width || frame.height !== height) throw new Error('GIF frame dimensions must match');
|
||||
const frameData = exactGifFrameData(frame.rgba) ?? quantizedGifFrameData(frame.rgba);
|
||||
const transparent = frameData.transparentIndex >= 0;
|
||||
gif.writeFrame(frameData.index, width, height, {
|
||||
palette: frameData.palette,
|
||||
delay: Math.max(0, Math.round(frame.delayMs || 0)),
|
||||
repeat: 0,
|
||||
transparent,
|
||||
transparentIndex: transparent ? frameData.transparentIndex : 0,
|
||||
first,
|
||||
});
|
||||
}
|
||||
|
||||
function encodeGifFrames(frames) {
|
||||
if (!frames.length) throw new Error('GIF encode requires at least one frame');
|
||||
const width = frames[0].width;
|
||||
const height = frames[0].height;
|
||||
const gif = GIFEncoder();
|
||||
for (const frame of frames) writeGifFrame(gif, frame, width, height, false);
|
||||
gif.finish();
|
||||
return gif.bytes();
|
||||
}
|
||||
|
||||
function encodeGifFrameChunk(frame, first) {
|
||||
const gif = GIFEncoder({auto: false});
|
||||
if (first) gif.writeHeader();
|
||||
writeGifFrame(gif, frame, frame.width, frame.height, first);
|
||||
return {data: gif.bytes(), width: frame.width, height: frame.height};
|
||||
}
|
||||
|
||||
function assembleGifFrameChunks(chunks) {
|
||||
if (!chunks.length) throw new Error('GIF chunk assembly requires at least one frame');
|
||||
const width = chunks[0].width;
|
||||
const height = chunks[0].height;
|
||||
const parts = [];
|
||||
for (const chunk of chunks) {
|
||||
if (chunk.width !== width || chunk.height !== height) throw new Error('GIF frame dimensions must match');
|
||||
parts.push(inputBytes(chunk.data));
|
||||
}
|
||||
parts.push(new Uint8Array([0x3b]));
|
||||
return concatBytes(parts);
|
||||
}
|
||||
|
||||
function writeU32BE(bytes, offset, value) {
|
||||
bytes[offset] = (value >>> 24) & 0xff;
|
||||
bytes[offset + 1] = (value >>> 16) & 0xff;
|
||||
bytes[offset + 2] = (value >>> 8) & 0xff;
|
||||
bytes[offset + 3] = value & 0xff;
|
||||
}
|
||||
|
||||
function writeU16BE(bytes, offset, value) {
|
||||
bytes[offset] = (value >>> 8) & 0xff;
|
||||
bytes[offset + 1] = value & 0xff;
|
||||
}
|
||||
|
||||
let pngCrcTable = null;
|
||||
function crc32(bytes, start, end) {
|
||||
if (!pngCrcTable) {
|
||||
pngCrcTable = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n += 1) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
pngCrcTable[n] = c >>> 0;
|
||||
}
|
||||
}
|
||||
let c = 0xffffffff;
|
||||
for (let index = start; index < end; index += 1) c = pngCrcTable[(c ^ bytes[index]) & 0xff] ^ (c >>> 8);
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function pngChunk(type, payload) {
|
||||
const typeBytes = textEncoder.encode(type);
|
||||
const chunk = new Uint8Array(12 + payload.length);
|
||||
writeU32BE(chunk, 0, payload.length);
|
||||
chunk.set(typeBytes, 4);
|
||||
chunk.set(payload, 8);
|
||||
writeU32BE(chunk, 8 + payload.length, crc32(chunk, 4, 8 + payload.length));
|
||||
return chunk;
|
||||
}
|
||||
|
||||
function concatBytes(parts) {
|
||||
let total = 0;
|
||||
for (const part of parts) total += part.length;
|
||||
const output = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
output.set(part, offset);
|
||||
offset += part.length;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function pngScanlines(rgba, width, height) {
|
||||
const rowBytes = width * 4;
|
||||
const output = new Uint8Array((rowBytes + 1) * height);
|
||||
for (let row = 0; row < height; row += 1) {
|
||||
const target = row * (rowBytes + 1);
|
||||
output[target] = 0;
|
||||
output.set(rgba.subarray(row * rowBytes, row * rowBytes + rowBytes), target + 1);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function delayFraction(delayMs) {
|
||||
const ms = Math.max(0, Math.round(Number(delayMs) || 0));
|
||||
if (ms === 0) return [0, 100];
|
||||
if (ms <= 655350) return [Math.min(65535, Math.max(1, Math.round(ms / 10))), 100];
|
||||
return [Math.min(65535, Math.max(1, Math.round(ms / 1000))), 1];
|
||||
}
|
||||
|
||||
function encodeApngFramePayload(frame) {
|
||||
return {
|
||||
compressed: zlibSync(pngScanlines(frame.rgba, frame.width, frame.height), {level: 6}),
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
delayMs: frame.delayMs,
|
||||
};
|
||||
}
|
||||
|
||||
function assembleApngFramePayloads(frames) {
|
||||
if (!frames.length) throw new Error('APNG encode requires at least one frame');
|
||||
const width = frames[0].width;
|
||||
const height = frames[0].height;
|
||||
const chunks = [new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])];
|
||||
const ihdr = new Uint8Array(13);
|
||||
writeU32BE(ihdr, 0, width);
|
||||
writeU32BE(ihdr, 4, height);
|
||||
ihdr[8] = 8;
|
||||
ihdr[9] = 6;
|
||||
chunks.push(pngChunk('IHDR', ihdr));
|
||||
const actl = new Uint8Array(8);
|
||||
writeU32BE(actl, 0, frames.length);
|
||||
writeU32BE(actl, 4, 0);
|
||||
chunks.push(pngChunk('acTL', actl));
|
||||
let sequence = 0;
|
||||
for (let index = 0; index < frames.length; index += 1) {
|
||||
const frame = frames[index];
|
||||
if (frame.width !== width || frame.height !== height) throw new Error('APNG frame dimensions must match');
|
||||
const fctl = new Uint8Array(26);
|
||||
writeU32BE(fctl, 0, sequence);
|
||||
sequence += 1;
|
||||
writeU32BE(fctl, 4, width);
|
||||
writeU32BE(fctl, 8, height);
|
||||
writeU32BE(fctl, 12, 0);
|
||||
writeU32BE(fctl, 16, 0);
|
||||
const delay = delayFraction(frame.delayMs);
|
||||
writeU16BE(fctl, 20, delay[0]);
|
||||
writeU16BE(fctl, 22, delay[1]);
|
||||
fctl[24] = 0;
|
||||
fctl[25] = 0;
|
||||
chunks.push(pngChunk('fcTL', fctl));
|
||||
if (index === 0) {
|
||||
chunks.push(pngChunk('IDAT', inputBytes(frame.compressed)));
|
||||
} else {
|
||||
const compressed = inputBytes(frame.compressed);
|
||||
const payload = new Uint8Array(4 + compressed.length);
|
||||
writeU32BE(payload, 0, sequence);
|
||||
sequence += 1;
|
||||
payload.set(compressed, 4);
|
||||
chunks.push(pngChunk('fdAT', payload));
|
||||
}
|
||||
}
|
||||
chunks.push(pngChunk('IEND', new Uint8Array()));
|
||||
return concatBytes(chunks);
|
||||
}
|
||||
|
||||
function encodeApngFrames(frames) {
|
||||
if (!frames.length) throw new Error('APNG encode requires at least one frame');
|
||||
const width = frames[0].width;
|
||||
const height = frames[0].height;
|
||||
for (const frame of frames) {
|
||||
if (frame.width !== width || frame.height !== height) throw new Error('APNG frame dimensions must match');
|
||||
}
|
||||
return assembleApngFramePayloads(frames.map(encodeApngFramePayload));
|
||||
}
|
||||
|
||||
function encodeStaticFrame(frame, outputFormat) {
|
||||
switch (outputFormat) {
|
||||
case 'png':
|
||||
return encodePng({width: frame.width, height: frame.height, data: frame.rgba, depth: 8, channels: 4});
|
||||
case 'jpeg':
|
||||
return encodeJpeg({width: frame.width, height: frame.height, data: frame.rgba}, 92).data;
|
||||
case 'gif':
|
||||
return encodeGifFrames([frame]);
|
||||
default:
|
||||
throw new Error(`Unsupported static output format: ${outputFormat}`);
|
||||
}
|
||||
}
|
||||
|
||||
function decodeStaticImage(input, inputFormat) {
|
||||
switch (inputFormat) {
|
||||
case 'png':
|
||||
return decodePngFrames(input)[0];
|
||||
case 'jpeg':
|
||||
return decodeJpegFrame(input);
|
||||
case 'gif':
|
||||
return decodeGifFrames(input)[0];
|
||||
default:
|
||||
throw new Error(`Unsupported static input format: ${inputFormat}`);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeFrameInput(frame) {
|
||||
if (!frame || frame.rgba == null) throw new Error('Frame is missing RGBA data');
|
||||
return {
|
||||
rgba: inputBytes(frame.rgba),
|
||||
width: nonNegativeU32(frame.width),
|
||||
height: nonNegativeU32(frame.height),
|
||||
delayMs: Math.max(0, Math.round(Number(frame.delayMs) || 0)),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeApngPayloadInput(frame) {
|
||||
if (!frame || frame.compressed == null) throw new Error('APNG frame is missing compressed data');
|
||||
return {
|
||||
compressed: inputBytes(frame.compressed),
|
||||
width: nonNegativeU32(frame.width),
|
||||
height: nonNegativeU32(frame.height),
|
||||
delayMs: Math.max(0, Math.round(Number(frame.delayMs) || 0)),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeGifChunkInput(chunk) {
|
||||
if (!chunk || chunk.data == null) throw new Error('GIF frame chunk is missing data');
|
||||
return {
|
||||
data: inputBytes(chunk.data),
|
||||
width: nonNegativeU32(chunk.width),
|
||||
height: nonNegativeU32(chunk.height),
|
||||
};
|
||||
}
|
||||
|
||||
export function decode_gif_frames(input) {
|
||||
return decodeGifFrames(input);
|
||||
}
|
||||
|
||||
export function decode_apng_frames(input) {
|
||||
return decodePngFrames(input);
|
||||
}
|
||||
|
||||
export function encode_gif_frames(frames) {
|
||||
return encodeGifFrames(frames.map(normalizeFrameInput));
|
||||
}
|
||||
|
||||
export function encode_apng_frames(frames) {
|
||||
return encodeApngFrames(frames.map(normalizeFrameInput));
|
||||
}
|
||||
|
||||
export function encode_gif_frame_chunk(frame, first) {
|
||||
return encodeGifFrameChunk(normalizeFrameInput(frame), Boolean(first));
|
||||
}
|
||||
|
||||
export function assemble_gif_frame_chunks(chunks) {
|
||||
return assembleGifFrameChunks(chunks.map(normalizeGifChunkInput));
|
||||
}
|
||||
|
||||
export function encode_apng_frame_payload(frame) {
|
||||
return encodeApngFramePayload(normalizeFrameInput(frame));
|
||||
}
|
||||
|
||||
export function assemble_apng_frames(frames) {
|
||||
return assembleApngFramePayloads(frames.map(normalizeApngPayloadInput));
|
||||
}
|
||||
|
||||
export function crop_and_rotate_apng(input, x, y, width, height, rotation_deg, resize_width, resize_height) {
|
||||
const bytes = inputBytes(input);
|
||||
const dimensions = pngDimensions(bytes);
|
||||
if (
|
||||
dimensions &&
|
||||
isNoopTransform(dimensions.width, dimensions.height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
const frames = decodePngFrames(bytes);
|
||||
if (
|
||||
isNoopTransform(frames[0].width, frames[0].height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
return encodeApngFrames(transformFrames(frames, x, y, width, height, rotation_deg, resize_width, resize_height));
|
||||
}
|
||||
|
||||
export function crop_and_rotate_gif(input, x, y, width, height, rotation_deg, resize_width, resize_height) {
|
||||
const bytes = inputBytes(input);
|
||||
const dimensions = gifDimensions(bytes);
|
||||
if (
|
||||
dimensions &&
|
||||
isNoopTransform(dimensions.width, dimensions.height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
return encodeGifFrames(transformGifFrames(bytes, x, y, width, height, rotation_deg, resize_width, resize_height));
|
||||
}
|
||||
|
||||
export function crop_and_rotate_image(
|
||||
input,
|
||||
format_hint,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotation_deg,
|
||||
resize_width,
|
||||
resize_height,
|
||||
) {
|
||||
const bytes = inputBytes(input);
|
||||
const inputFormat = sniffImageFormat(bytes);
|
||||
const requestedFormat = normalizedFormat(format_hint);
|
||||
const outputFormat = requestedFormat && requestedFormat !== 'unknown' ? requestedFormat : inputFormat;
|
||||
if (inputFormat === 'webp' || inputFormat === 'avif' || outputFormat === 'webp' || outputFormat === 'avif') {
|
||||
throw new Error('WebP and AVIF crop/encode use the browser or native media bridge');
|
||||
}
|
||||
if (inputFormat === outputFormat) {
|
||||
const dimensions = imageDimensions(bytes, inputFormat);
|
||||
if (
|
||||
dimensions &&
|
||||
isNoopTransform(
|
||||
dimensions.width,
|
||||
dimensions.height,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotation_deg,
|
||||
resize_width,
|
||||
resize_height,
|
||||
)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
}
|
||||
const frame = decodeStaticImage(bytes, inputFormat);
|
||||
if (
|
||||
inputFormat === outputFormat &&
|
||||
isNoopTransform(frame.width, frame.height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
return encodeStaticFrame(
|
||||
transformFrame(frame, x, y, width, height, rotation_deg, resize_width, resize_height),
|
||||
outputFormat,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "fluxer-content-update-frozen-snapshot"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.102"
|
||||
base64 = "0.22.1"
|
||||
sha2 = "0.11.0"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.27.0"
|
||||
@@ -0,0 +1,158 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use base64::Engine;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn generate_snapshot_source(static_dir: &Path) -> Result<String> {
|
||||
if !static_dir.is_dir() {
|
||||
bail!("{} is not a directory", static_dir.display());
|
||||
}
|
||||
|
||||
let index_html = read_file(&static_dir.join("index.html"))?;
|
||||
let sw_js = read_file(&static_dir.join("sw.js"))?;
|
||||
let version_json = read_file(&static_dir.join("version.json"))?;
|
||||
let sha = snapshot_sha(&index_html, &sw_js, &version_json);
|
||||
|
||||
let mut output = String::new();
|
||||
output.push_str(&format!("const SNAPSHOT_SHA: &str = \"{sha}\";\n\n"));
|
||||
output.push_str(&format!(
|
||||
"// -- base64-encoded index.html ({} bytes) --\n",
|
||||
index_html.len()
|
||||
));
|
||||
output.push_str(&format_const(
|
||||
"STABLE_INDEX_HTML",
|
||||
&base64_standard(&index_html),
|
||||
));
|
||||
output.push_str(&format!(
|
||||
"// -- base64-encoded sw.js ({} bytes) --\n",
|
||||
sw_js.len()
|
||||
));
|
||||
output.push_str(&format_const("STABLE_SW_JS", &base64_standard(&sw_js)));
|
||||
output.push_str(&format!(
|
||||
"// -- base64-encoded version.json ({} bytes) --\n",
|
||||
version_json.len()
|
||||
));
|
||||
output.push_str(&format_const(
|
||||
"STABLE_VERSION_JSON",
|
||||
&base64_standard(&version_json),
|
||||
));
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn read_file(path: &Path) -> Result<Vec<u8>> {
|
||||
if !path.exists() {
|
||||
bail!("{} does not exist", path.display());
|
||||
}
|
||||
fs::read(path).with_context(|| format!("failed to read {}", path.display()))
|
||||
}
|
||||
|
||||
fn snapshot_sha(index_html: &[u8], sw_js: &[u8], version_json: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(index_html);
|
||||
hasher.update(sw_js);
|
||||
hasher.update(version_json);
|
||||
bytes_to_lower_hex(&hasher.finalize())
|
||||
}
|
||||
|
||||
fn bytes_to_lower_hex(bytes: &[u8]) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut out = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
out.push(HEX[(byte >> 4) as usize] as char);
|
||||
out.push(HEX[(byte & 0x0f) as usize] as char);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn base64_standard(data: &[u8]) -> String {
|
||||
base64::engine::general_purpose::STANDARD.encode(data)
|
||||
}
|
||||
|
||||
fn break_lines(value: &str, width: usize) -> Vec<&str> {
|
||||
value
|
||||
.as_bytes()
|
||||
.chunks(width)
|
||||
.map(|chunk| std::str::from_utf8(chunk).expect("base64 should be ascii"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn format_const(name: &str, data: &str) -> String {
|
||||
let escaped = break_lines(data, 100).join("\\\n");
|
||||
format!("const {name}: &str = \"\\\n{escaped}\\\n\";\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn emits_snapshot_constants_with_combined_sha() {
|
||||
let dir = tempdir().unwrap();
|
||||
let index_html = b"<!doctype html><html><body>fixture</body></html>";
|
||||
let sw_js = b"\"use strict\";\nself.addEventListener('install', () => {});\n";
|
||||
let version_json = br#"{"sha":"fixture","buildNumber":7}"#;
|
||||
fs::write(dir.path().join("index.html"), index_html).unwrap();
|
||||
fs::write(dir.path().join("sw.js"), sw_js).unwrap();
|
||||
fs::write(dir.path().join("version.json"), version_json).unwrap();
|
||||
|
||||
let output = generate_snapshot_source(dir.path()).unwrap();
|
||||
let expected_sha = snapshot_sha(index_html, sw_js, version_json);
|
||||
|
||||
assert!(output.starts_with(&format!(
|
||||
"const SNAPSHOT_SHA: &str = \"{expected_sha}\";\n\n"
|
||||
)));
|
||||
assert!(output.contains("// -- base64-encoded index.html (48 bytes) --"));
|
||||
assert!(output.contains("// -- base64-encoded sw.js (58 bytes) --"));
|
||||
assert!(output.contains("// -- base64-encoded version.json (33 bytes) --"));
|
||||
assert!(output.contains(&format_const(
|
||||
"STABLE_INDEX_HTML",
|
||||
&base64_standard(index_html)
|
||||
)));
|
||||
assert!(output.contains(&format_const("STABLE_SW_JS", &base64_standard(sw_js))));
|
||||
assert!(output.contains(&format_const(
|
||||
"STABLE_VERSION_JSON",
|
||||
&base64_standard(version_json)
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wraps_base64_constants_at_100_columns() {
|
||||
let data = "a".repeat(205);
|
||||
let formatted = format_const("TEST", &data);
|
||||
let lines = formatted.lines().collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(lines[0], "const TEST: &str = \"\\");
|
||||
assert_eq!(lines[1], format!("{}\\", "a".repeat(100)));
|
||||
assert_eq!(lines[2], format!("{}\\", "a".repeat(100)));
|
||||
assert_eq!(lines[3], format!("{}\\", "a".repeat(5)));
|
||||
assert_eq!(lines[4], "\";");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_non_directory_static_path() {
|
||||
let dir = tempdir().unwrap();
|
||||
let err = generate_snapshot_source(&dir.path().join("missing"))
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
|
||||
assert!(err.contains("missing is not a directory"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_missing_required_file() {
|
||||
let dir = tempdir().unwrap();
|
||||
fs::write(dir.path().join("index.html"), b"index").unwrap();
|
||||
|
||||
let err = generate_snapshot_source(dir.path())
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
|
||||
assert!(err.contains("sw.js does not exist"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::Result;
|
||||
use fluxer_content_update_frozen_snapshot::generate_snapshot_source;
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use std::process;
|
||||
|
||||
fn main() {
|
||||
if let Err(err) = run() {
|
||||
eprintln!("error: {err:#}");
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<()> {
|
||||
let mut args = env::args_os();
|
||||
let program = args
|
||||
.next()
|
||||
.and_then(|value| PathBuf::from(value).file_name().map(|name| name.to_owned()))
|
||||
.and_then(|name| name.into_string().ok())
|
||||
.unwrap_or_else(|| "fluxer-content-update-frozen-snapshot".to_owned());
|
||||
|
||||
let Some(static_dir) = args.next() else {
|
||||
eprintln!("usage: {program} <static_dir>");
|
||||
process::exit(1);
|
||||
};
|
||||
if args.next().is_some() {
|
||||
eprintln!("usage: {program} <static_dir>");
|
||||
process::exit(1);
|
||||
}
|
||||
|
||||
let output = generate_snapshot_source(&PathBuf::from(static_dir))?;
|
||||
print!("{output}");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "fluxer-dev"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.102"
|
||||
axum = "0.8.9"
|
||||
base64 = "0.22.1"
|
||||
chrono = "0.4.45"
|
||||
clap = { version = "4.6.1", features = ["derive", "env"] }
|
||||
futures-util = "0.3.32"
|
||||
hmac = "0.13.0"
|
||||
hyper = { version = "1.10.1", features = ["http1", "server"] }
|
||||
hyper-util = { version = "0.1.20", features = ["tokio"] }
|
||||
image = { version = "0.25.10", default-features = false, features = ["gif", "jpeg", "png", "webp"] }
|
||||
libc = "0.2.186"
|
||||
regex = "1.12"
|
||||
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] }
|
||||
scylla = { version = "1.6.0", features = ["chrono-04"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
sha2 = "0.11.0"
|
||||
tempfile = "3.27.0"
|
||||
tokio = { version = "1.52.3", features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "time"] }
|
||||
tokio-tungstenite = { version = "0.29.0", features = ["rustls-tls-native-roots"] }
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
|
||||
[dev-dependencies]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::desktop::install_desktop;
|
||||
use crate::gateway::setup_gateway_config;
|
||||
use crate::paths::{ensure_state_dirs, ensure_writable_dev_paths};
|
||||
use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http, wait_tcp};
|
||||
use crate::smoke::{bootstrap_schema_and_object_store, run_smoke, wait_s3_api};
|
||||
use anyhow::Result;
|
||||
|
||||
pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result<()> {
|
||||
ensure_state_dirs()?;
|
||||
ensure_writable_dev_paths()?;
|
||||
if !skip_install {
|
||||
crate::proc::run(&["corepack", "enable"])?;
|
||||
crate::proc::run(&["corepack", "prepare", "pnpm@10.29.3", "--activate"])?;
|
||||
run_command(
|
||||
&["pnpm", "install", "--frozen-lockfile"],
|
||||
RunOptions {
|
||||
env: PNPM_INSTALL_ENV
|
||||
.iter()
|
||||
.map(|(key, value)| ((*key).to_owned(), Some((*value).to_owned())))
|
||||
.collect(),
|
||||
..RunOptions::default()
|
||||
},
|
||||
)?;
|
||||
if !skip_desktop_install {
|
||||
install_desktop()?;
|
||||
}
|
||||
}
|
||||
setup_gateway_config()?;
|
||||
wait_core_infra().await?;
|
||||
bootstrap_schema_and_object_store().await?;
|
||||
run_smoke(false, false).await?;
|
||||
println!("Fluxer dev bootstrap complete.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn post_start() -> Result<()> {
|
||||
ensure_state_dirs()?;
|
||||
ensure_writable_dev_paths()?;
|
||||
setup_gateway_config()?;
|
||||
run_smoke(true, false).await
|
||||
}
|
||||
|
||||
pub async fn wait_core_infra() -> Result<()> {
|
||||
wait_tcp("Valkey", "valkey", 6379, 120).await?;
|
||||
wait_tcp("NATS", "nats", 4222, 120).await?;
|
||||
wait_tcp("LiveKit", "livekit", 7880, 120).await?;
|
||||
crate::media_proxy::ensure_dev_object_store(true, 120).await?;
|
||||
wait_tcp("SeaweedFS S3", "127.0.0.1", 8333, 120).await?;
|
||||
wait_http(
|
||||
"SeaweedFS master",
|
||||
"http://127.0.0.1:9333/cluster/status",
|
||||
120,
|
||||
)
|
||||
.await?;
|
||||
wait_s3_api(120).await
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::paths::DEV_CASSANDRA_DIR;
|
||||
use anyhow::{Context, Result, bail};
|
||||
use regex::Regex;
|
||||
use scylla::DeserializeRow;
|
||||
use scylla::client::session::Session;
|
||||
use scylla::client::session_builder::SessionBuilder;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::LazyLock;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CassandraConfig {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub keyspace: String,
|
||||
pub local_dc: String,
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ExistingTable {
|
||||
pub columns: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ExistingSchema {
|
||||
pub keyspace_exists: bool,
|
||||
pub types: BTreeMap<String, BTreeMap<String, String>>,
|
||||
pub tables: BTreeMap<String, ExistingTable>,
|
||||
pub indexes: BTreeSet<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SchemaDiff {
|
||||
pub statements: Vec<String>,
|
||||
pub warnings: Vec<String>,
|
||||
pub errors: Vec<String>,
|
||||
pub target_sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct Field {
|
||||
pub name: String,
|
||||
#[serde(rename = "type")]
|
||||
pub field_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct UserType {
|
||||
pub name: String,
|
||||
pub fields: Vec<Field>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct Table {
|
||||
pub name: String,
|
||||
pub columns: Vec<Field>,
|
||||
pub primary_key: String,
|
||||
pub options: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct Index {
|
||||
pub name: String,
|
||||
pub table: String,
|
||||
pub expression: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
pub struct TargetSchema {
|
||||
pub user_types: Vec<UserType>,
|
||||
pub tables: Vec<Table>,
|
||||
pub indexes: Vec<Index>,
|
||||
}
|
||||
|
||||
static TARGET_SCHEMA: LazyLock<TargetSchema> = LazyLock::new(|| {
|
||||
serde_json::from_str(include_str!("../cassandra_target_schema.json"))
|
||||
.expect("embedded Cassandra target schema JSON is valid")
|
||||
});
|
||||
|
||||
static TYPE_TOKEN_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"[a-zA-Z_][a-zA-Z0-9_]*").expect("valid type token regex"));
|
||||
static WHITESPACE_RE: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"\s+").expect("valid whitespace regex"));
|
||||
|
||||
pub fn target_schema() -> &'static TargetSchema {
|
||||
&TARGET_SCHEMA
|
||||
}
|
||||
|
||||
pub fn config_from_env() -> Result<CassandraConfig> {
|
||||
let hosts = env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned());
|
||||
let host = hosts
|
||||
.split_once(',')
|
||||
.map(|(first, _)| first)
|
||||
.unwrap_or(&hosts)
|
||||
.trim();
|
||||
Ok(CassandraConfig {
|
||||
host: if host.is_empty() {
|
||||
"cassandra".to_owned()
|
||||
} else {
|
||||
host.to_owned()
|
||||
},
|
||||
port: env::var("FLUXER_CASSANDRA_PORT")
|
||||
.unwrap_or_else(|_| "9042".to_owned())
|
||||
.parse()
|
||||
.context("invalid FLUXER_CASSANDRA_PORT")?,
|
||||
keyspace: env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned()),
|
||||
local_dc: env::var("FLUXER_CASSANDRA_LOCAL_DC")
|
||||
.unwrap_or_else(|_| "datacenter1".to_owned()),
|
||||
username: env::var("FLUXER_CASSANDRA_USERNAME")
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty()),
|
||||
password: env::var("FLUXER_CASSANDRA_PASSWORD")
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty()),
|
||||
})
|
||||
}
|
||||
|
||||
async fn connect(config: &CassandraConfig) -> Result<Session> {
|
||||
let contact = format!("{}:{}", config.host, config.port);
|
||||
let mut builder = SessionBuilder::new().known_node(contact);
|
||||
if let Some(username) = &config.username {
|
||||
builder = builder.user(username, config.password.as_deref().unwrap_or_default());
|
||||
}
|
||||
Ok(builder.build().await?)
|
||||
}
|
||||
|
||||
pub async fn wait_for_cassandra(config: &CassandraConfig, timeout_secs: u64) -> Result<()> {
|
||||
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
|
||||
let mut last_error = None;
|
||||
while Instant::now() < deadline {
|
||||
match connect(config).await {
|
||||
Ok(session) => match session
|
||||
.query_unpaged("SELECT release_version FROM system.local", ())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
println!("Cassandra is reachable at {}:{}", config.host, config.port);
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => last_error = Some(error.to_string()),
|
||||
},
|
||||
Err(error) => last_error = Some(error.to_string()),
|
||||
}
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
bail!(
|
||||
"Timed out waiting for Cassandra at {}:{}: {}",
|
||||
config.host,
|
||||
config.port,
|
||||
last_error.unwrap_or_else(|| "unknown error".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
pub fn normalize_type(value: &str) -> String {
|
||||
let mut normalized = WHITESPACE_RE.replace_all(value, "").to_ascii_lowercase();
|
||||
let user_type_names = target_schema()
|
||||
.user_types
|
||||
.iter()
|
||||
.map(|user_type| user_type.name.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
let mut previous = String::new();
|
||||
while previous != normalized {
|
||||
previous = normalized.clone();
|
||||
for name in &user_type_names {
|
||||
normalized = normalized.replace(&format!("frozen<{name}>"), name);
|
||||
}
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
fn normalize_identifier(value: &str) -> String {
|
||||
value.trim_matches('"').to_ascii_lowercase()
|
||||
}
|
||||
|
||||
#[derive(Debug, DeserializeRow)]
|
||||
#[allow(dead_code)]
|
||||
struct KeyspaceRow {
|
||||
keyspace_name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, DeserializeRow)]
|
||||
struct TypeRow {
|
||||
type_name: String,
|
||||
field_names: Option<Vec<String>>,
|
||||
field_types: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, DeserializeRow)]
|
||||
struct ColumnRow {
|
||||
table_name: String,
|
||||
column_name: String,
|
||||
#[scylla(rename = "type")]
|
||||
column_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, DeserializeRow)]
|
||||
struct IndexRow {
|
||||
index_name: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn fetch_existing_schema(session: &Session, keyspace: &str) -> Result<ExistingSchema> {
|
||||
let keyspace_rows = session
|
||||
.query_unpaged(
|
||||
"SELECT keyspace_name FROM system_schema.keyspaces WHERE keyspace_name = ?",
|
||||
(keyspace,),
|
||||
)
|
||||
.await?
|
||||
.into_rows_result()?
|
||||
.rows::<KeyspaceRow>()?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
|
||||
let type_rows = session
|
||||
.query_unpaged(
|
||||
"SELECT type_name, field_names, field_types FROM system_schema.types WHERE keyspace_name = ?",
|
||||
(keyspace,),
|
||||
)
|
||||
.await?
|
||||
.into_rows_result()?
|
||||
.rows::<TypeRow>()?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
let mut types = BTreeMap::new();
|
||||
for row in type_rows {
|
||||
let mut fields = BTreeMap::new();
|
||||
for (name, field_type) in row
|
||||
.field_names
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.zip(row.field_types.unwrap_or_default())
|
||||
{
|
||||
fields.insert(normalize_identifier(&name), field_type);
|
||||
}
|
||||
types.insert(normalize_identifier(&row.type_name), fields);
|
||||
}
|
||||
|
||||
let column_rows = session
|
||||
.query_unpaged(
|
||||
"SELECT table_name, column_name, type FROM system_schema.columns WHERE keyspace_name = ?",
|
||||
(keyspace,),
|
||||
)
|
||||
.await?
|
||||
.into_rows_result()?
|
||||
.rows::<ColumnRow>()?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
let mut tables: BTreeMap<String, ExistingTable> = BTreeMap::new();
|
||||
for row in column_rows {
|
||||
tables
|
||||
.entry(normalize_identifier(&row.table_name))
|
||||
.or_default()
|
||||
.columns
|
||||
.insert(normalize_identifier(&row.column_name), row.column_type);
|
||||
}
|
||||
|
||||
let index_rows = session
|
||||
.query_unpaged(
|
||||
"SELECT index_name FROM system_schema.indexes WHERE keyspace_name = ?",
|
||||
(keyspace,),
|
||||
)
|
||||
.await?
|
||||
.into_rows_result()?
|
||||
.rows::<IndexRow>()?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
let indexes = index_rows
|
||||
.into_iter()
|
||||
.filter_map(|row| row.index_name)
|
||||
.map(|name| normalize_identifier(&name))
|
||||
.collect();
|
||||
|
||||
Ok(ExistingSchema {
|
||||
keyspace_exists: !keyspace_rows.is_empty(),
|
||||
types,
|
||||
tables,
|
||||
indexes,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn render_user_type(user_type: &UserType, keyspace: &str) -> String {
|
||||
let fields = user_type
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| format!("{} {}", field.name, field.field_type))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",\n ");
|
||||
format!(
|
||||
"CREATE TYPE IF NOT EXISTS {keyspace}.{} (\n {fields}\n)",
|
||||
user_type.name
|
||||
)
|
||||
}
|
||||
|
||||
pub fn render_table(table: &Table, keyspace: &str) -> String {
|
||||
let mut fields = table
|
||||
.columns
|
||||
.iter()
|
||||
.map(|field| format!("{} {}", field.name, field.field_type))
|
||||
.collect::<Vec<_>>();
|
||||
fields.push(format!("PRIMARY KEY {}", table.primary_key));
|
||||
let body = fields.join(",\n ");
|
||||
let mut statement = format!(
|
||||
"CREATE TABLE IF NOT EXISTS {keyspace}.{} (\n {body}\n)",
|
||||
table.name
|
||||
);
|
||||
if !table.options.is_empty() {
|
||||
statement.push_str(" WITH ");
|
||||
statement.push_str(&table.options);
|
||||
}
|
||||
statement
|
||||
}
|
||||
|
||||
pub fn render_index(index: &Index, keyspace: &str) -> String {
|
||||
format!(
|
||||
"CREATE INDEX IF NOT EXISTS {} ON {keyspace}.{} ({})",
|
||||
index.name, index.table, index.expression
|
||||
)
|
||||
}
|
||||
|
||||
pub fn render_target_schema(keyspace: &str) -> String {
|
||||
let schema = target_schema();
|
||||
let mut statements = Vec::new();
|
||||
for user_type in sorted_user_types() {
|
||||
statements.push(statement_with_semicolon(&render_user_type(
|
||||
&user_type, keyspace,
|
||||
)));
|
||||
}
|
||||
let mut tables = schema.tables.clone();
|
||||
tables.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
for table in tables {
|
||||
statements.push(statement_with_semicolon(&render_table(&table, keyspace)));
|
||||
}
|
||||
let mut indexes = schema.indexes.clone();
|
||||
indexes.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
for index in indexes {
|
||||
statements.push(statement_with_semicolon(&render_index(&index, keyspace)));
|
||||
}
|
||||
format!("{}\n", statements.join("\n\n"))
|
||||
}
|
||||
|
||||
pub fn target_schema_sha256(keyspace: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(render_target_schema(keyspace).as_bytes());
|
||||
bytes_to_lower_hex(&hasher.finalize())
|
||||
}
|
||||
|
||||
fn bytes_to_lower_hex(bytes: &[u8]) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut out = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
out.push(HEX[(byte >> 4) as usize] as char);
|
||||
out.push(HEX[(byte & 0x0f) as usize] as char);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn sorted_user_types() -> Vec<UserType> {
|
||||
let schema = target_schema();
|
||||
let by_name = schema
|
||||
.user_types
|
||||
.iter()
|
||||
.map(|user_type| (user_type.name.clone(), user_type.clone()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut done = BTreeSet::new();
|
||||
let mut result = Vec::new();
|
||||
let mut remaining = by_name.clone();
|
||||
while !remaining.is_empty() {
|
||||
let mut progressed = false;
|
||||
for (name, user_type) in remaining.clone() {
|
||||
let deps = type_dependencies(&user_type, &by_name.keys().cloned().collect());
|
||||
if deps.is_subset(&done) {
|
||||
result.push(user_type);
|
||||
done.insert(name.clone());
|
||||
remaining.remove(&name);
|
||||
progressed = true;
|
||||
}
|
||||
}
|
||||
if !progressed {
|
||||
result.extend(remaining.into_values());
|
||||
break;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn type_dependencies(user_type: &UserType, user_type_names: &BTreeSet<String>) -> BTreeSet<String> {
|
||||
let mut deps = BTreeSet::new();
|
||||
for field in &user_type.fields {
|
||||
for token in TYPE_TOKEN_RE.find_iter(&field.field_type) {
|
||||
let token = token.as_str().to_ascii_lowercase();
|
||||
if user_type_names.contains(&token) && token != user_type.name {
|
||||
deps.insert(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
deps
|
||||
}
|
||||
|
||||
pub fn create_diff(existing: &ExistingSchema, keyspace: &str) -> SchemaDiff {
|
||||
let schema = target_schema();
|
||||
let mut statements = Vec::new();
|
||||
let mut warnings = Vec::new();
|
||||
let mut errors = Vec::new();
|
||||
if !existing.keyspace_exists {
|
||||
statements.push(format!(
|
||||
"CREATE KEYSPACE IF NOT EXISTS {keyspace} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}"
|
||||
));
|
||||
}
|
||||
for user_type in sorted_user_types() {
|
||||
let Some(current_fields) = existing.types.get(&user_type.name) else {
|
||||
statements.push(render_user_type(&user_type, keyspace));
|
||||
continue;
|
||||
};
|
||||
for field in &user_type.fields {
|
||||
match current_fields.get(&field.name) {
|
||||
None => statements.push(format!(
|
||||
"ALTER TYPE {keyspace}.{} ADD IF NOT EXISTS {} {}",
|
||||
user_type.name, field.name, field.field_type
|
||||
)),
|
||||
Some(current_type)
|
||||
if normalize_type(current_type) != normalize_type(&field.field_type) =>
|
||||
{
|
||||
errors.push(format!(
|
||||
"type {}.{} has {current_type:?}, expected {:?}",
|
||||
user_type.name, field.name, field.field_type
|
||||
));
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut tables = schema.tables.clone();
|
||||
tables.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
for table in tables {
|
||||
let Some(current_table) = existing.tables.get(&table.name) else {
|
||||
statements.push(render_table(&table, keyspace));
|
||||
continue;
|
||||
};
|
||||
for field in &table.columns {
|
||||
match current_table.columns.get(&field.name) {
|
||||
None => statements.push(format!(
|
||||
"ALTER TABLE {keyspace}.{} ADD IF NOT EXISTS {} {}",
|
||||
table.name, field.name, field.field_type
|
||||
)),
|
||||
Some(current_type)
|
||||
if normalize_type(current_type) != normalize_type(&field.field_type) =>
|
||||
{
|
||||
errors.push(format!(
|
||||
"table {}.{} has {current_type:?}, expected {:?}",
|
||||
table.name, field.name, field.field_type
|
||||
));
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut indexes = schema.indexes.clone();
|
||||
indexes.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
for index in indexes {
|
||||
if !existing.indexes.contains(&index.name.to_ascii_lowercase()) {
|
||||
statements.push(render_index(&index, keyspace));
|
||||
}
|
||||
}
|
||||
let target_tables = schema
|
||||
.tables
|
||||
.iter()
|
||||
.map(|table| table.name.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
for table_name in existing.tables.keys() {
|
||||
if !target_tables.contains(table_name) && !table_name.starts_with("system_") {
|
||||
warnings.push(format!(
|
||||
"extra Cassandra table exists outside target schema: {table_name}"
|
||||
));
|
||||
}
|
||||
}
|
||||
SchemaDiff {
|
||||
statements,
|
||||
warnings,
|
||||
errors,
|
||||
target_sha256: target_schema_sha256(keyspace),
|
||||
}
|
||||
}
|
||||
|
||||
fn statement_with_semicolon(statement: &str) -> String {
|
||||
format!("{};", statement.trim().trim_end_matches(';'))
|
||||
}
|
||||
|
||||
pub fn render_diff(diff: &SchemaDiff) -> String {
|
||||
let mut lines = vec![
|
||||
"-- Generated by fluxer-dev cassandra diff".to_owned(),
|
||||
format!("-- Target schema sha256: {}", diff.target_sha256),
|
||||
String::new(),
|
||||
];
|
||||
if !diff.errors.is_empty() {
|
||||
lines.push("-- Diff contains validation errors and should not be applied:".to_owned());
|
||||
lines.extend(diff.errors.iter().map(|error| format!("-- {error}")));
|
||||
lines.push(String::new());
|
||||
}
|
||||
if !diff.warnings.is_empty() {
|
||||
lines.push("-- Warnings:".to_owned());
|
||||
lines.extend(
|
||||
diff.warnings
|
||||
.iter()
|
||||
.map(|warning| format!("-- {warning}")),
|
||||
);
|
||||
lines.push(String::new());
|
||||
}
|
||||
if diff.statements.is_empty() {
|
||||
lines.push("-- No schema changes required.".to_owned());
|
||||
} else {
|
||||
lines.extend(
|
||||
diff.statements
|
||||
.iter()
|
||||
.map(|statement| statement_with_semicolon(statement)),
|
||||
);
|
||||
}
|
||||
format!("{}\n", lines.join("\n"))
|
||||
}
|
||||
|
||||
pub fn write_diff_file(diff: &SchemaDiff, path: Option<&Path>) -> Result<PathBuf> {
|
||||
std::fs::create_dir_all(DEV_CASSANDRA_DIR.as_path())?;
|
||||
let output = path
|
||||
.map(Path::to_path_buf)
|
||||
.unwrap_or_else(|| DEV_CASSANDRA_DIR.join("last-diff.cql"));
|
||||
std::fs::write(&output, render_diff(diff))?;
|
||||
std::fs::write(
|
||||
DEV_CASSANDRA_DIR.join("target-schema.sha256"),
|
||||
format!("{}\n", diff.target_sha256),
|
||||
)?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
pub async fn compute_diff(config: Option<CassandraConfig>) -> Result<SchemaDiff> {
|
||||
let config = config.unwrap_or(config_from_env()?);
|
||||
wait_for_cassandra(&config, 180).await?;
|
||||
let session = connect(&config).await?;
|
||||
create_live_diff(&session, &config.keyspace).await
|
||||
}
|
||||
|
||||
async fn create_live_diff(session: &Session, keyspace: &str) -> Result<SchemaDiff> {
|
||||
Ok(create_diff(
|
||||
&fetch_existing_schema(session, keyspace).await?,
|
||||
keyspace,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn apply_schema(config: Option<CassandraConfig>) -> Result<SchemaDiff> {
|
||||
let config = config.unwrap_or(config_from_env()?);
|
||||
wait_for_cassandra(&config, 180).await?;
|
||||
let session = connect(&config).await?;
|
||||
let diff = create_live_diff(&session, &config.keyspace).await?;
|
||||
let diff_path = write_diff_file(&diff, None)?;
|
||||
if !diff.errors.is_empty() {
|
||||
bail!(
|
||||
"Cassandra schema has validation errors; wrote {}",
|
||||
diff_path.display()
|
||||
);
|
||||
}
|
||||
for warning in &diff.warnings {
|
||||
println!("Cassandra schema warning: {warning}");
|
||||
}
|
||||
if diff.statements.is_empty() {
|
||||
println!(
|
||||
"Cassandra schema is already up to date; wrote {}",
|
||||
diff_path.display()
|
||||
);
|
||||
} else {
|
||||
println!(
|
||||
"Applying {} Cassandra schema statement(s); wrote {}",
|
||||
diff.statements.len(),
|
||||
diff_path.display()
|
||||
);
|
||||
for statement in &diff.statements {
|
||||
session.query_unpaged(statement.as_str(), ()).await?;
|
||||
}
|
||||
}
|
||||
verify_schema_with_session(&session, &config.keyspace).await?;
|
||||
Ok(diff)
|
||||
}
|
||||
|
||||
pub async fn verify_schema(
|
||||
config: Option<CassandraConfig>,
|
||||
keyspace: Option<String>,
|
||||
) -> Result<()> {
|
||||
let config = config.unwrap_or(config_from_env()?);
|
||||
let keyspace = keyspace.unwrap_or_else(|| config.keyspace.clone());
|
||||
wait_for_cassandra(&config, 180).await?;
|
||||
let session = connect(&config).await?;
|
||||
verify_schema_with_session(&session, &keyspace).await
|
||||
}
|
||||
|
||||
async fn verify_schema_with_session(session: &Session, keyspace: &str) -> Result<()> {
|
||||
let existing = fetch_existing_schema(session, keyspace).await?;
|
||||
let diff = create_diff(&existing, keyspace);
|
||||
let unapplied = diff.statements.clone();
|
||||
if !diff.errors.is_empty() || !unapplied.is_empty() {
|
||||
write_diff_file(&diff, None)?;
|
||||
let mut problems = diff.errors;
|
||||
problems.extend(
|
||||
unapplied
|
||||
.into_iter()
|
||||
.take(10)
|
||||
.map(|statement| format!("still needs: {statement}")),
|
||||
);
|
||||
bail!(
|
||||
"Cassandra schema verification failed:\n{}",
|
||||
problems
|
||||
.into_iter()
|
||||
.map(|item| format!(" - {item}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"Cassandra schema verified: {} types, {} tables, {} indexes, target {}",
|
||||
target_schema().user_types.len(),
|
||||
target_schema().tables.len(),
|
||||
target_schema().indexes.len(),
|
||||
&diff.target_sha256[..12],
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn normalizes_frozen_user_type_wrappers() {
|
||||
assert_eq!(normalize_type(" frozen<message_embed> "), "message_embed");
|
||||
assert_eq!(
|
||||
normalize_type("LIST< frozen<message_embed> >"),
|
||||
"list<message_embed>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_target_schema_with_semicolons() {
|
||||
let rendered = render_target_schema("fluxer");
|
||||
assert!(rendered.starts_with("CREATE TYPE IF NOT EXISTS fluxer."));
|
||||
assert!(rendered.contains("CREATE TABLE IF NOT EXISTS fluxer.messages"));
|
||||
assert!(rendered.ends_with(";\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_creates_keyspace_and_missing_schema() {
|
||||
let diff = create_diff(&ExistingSchema::default(), "fluxer");
|
||||
assert!(diff.statements[0].starts_with("CREATE KEYSPACE IF NOT EXISTS fluxer"));
|
||||
assert!(
|
||||
diff.statements
|
||||
.iter()
|
||||
.any(|statement| statement.starts_with("CREATE TYPE IF NOT EXISTS fluxer."))
|
||||
);
|
||||
assert!(
|
||||
diff.statements
|
||||
.iter()
|
||||
.any(|statement| statement.starts_with("CREATE TABLE IF NOT EXISTS fluxer."))
|
||||
);
|
||||
assert!(diff.errors.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_reports_type_mismatches_and_extra_tables() {
|
||||
let mut existing = ExistingSchema {
|
||||
keyspace_exists: true,
|
||||
..ExistingSchema::default()
|
||||
};
|
||||
existing.types.insert(
|
||||
"message_attachment".to_owned(),
|
||||
BTreeMap::from([("attachment_id".to_owned(), "text".to_owned())]),
|
||||
);
|
||||
existing
|
||||
.tables
|
||||
.insert("unexpected".to_owned(), ExistingTable::default());
|
||||
let diff = create_diff(&existing, "fluxer");
|
||||
assert!(
|
||||
diff.errors
|
||||
.iter()
|
||||
.any(|error| error.contains("type message_attachment.attachment_id"))
|
||||
);
|
||||
assert!(
|
||||
diff.warnings
|
||||
.iter()
|
||||
.any(|warning| warning.contains("unexpected"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_diff_includes_warnings_errors_and_sha() {
|
||||
let diff = SchemaDiff {
|
||||
statements: vec!["SELECT 1".to_owned()],
|
||||
warnings: vec!["warn".to_owned()],
|
||||
errors: vec!["err".to_owned()],
|
||||
target_sha256: "abc".to_owned(),
|
||||
};
|
||||
let rendered = render_diff(&diff);
|
||||
assert!(rendered.contains("-- Target schema sha256: abc"));
|
||||
assert!(rendered.contains("-- warn"));
|
||||
assert!(rendered.contains("-- err"));
|
||||
assert!(rendered.ends_with("SELECT 1;\n"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,601 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::manifest::LOCAL_APP_URL;
|
||||
use crate::paths::{DESKTOP_DIR, ROOT};
|
||||
use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command};
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use url::Url;
|
||||
|
||||
const CANARY_APP_NAME: &str = "Fluxer Canary";
|
||||
const CANARY_BUNDLE_ID: &str = "app.fluxer.canary";
|
||||
const CANARY_RPC_PORT: u16 = 21864;
|
||||
const MACOS_DEV_ELECTRON_USAGE_DESCRIPTIONS: &[(&str, &str)] = &[
|
||||
(
|
||||
"NSMicrophoneUsageDescription",
|
||||
"Fluxer needs access to your microphone to enable voice chat features.",
|
||||
),
|
||||
(
|
||||
"NSCameraUsageDescription",
|
||||
"Fluxer needs access to your camera to enable video chat features.",
|
||||
),
|
||||
(
|
||||
"NSAppleEventsUsageDescription",
|
||||
"Fluxer needs access to Apple Events for automation features.",
|
||||
),
|
||||
(
|
||||
"NSAudioCaptureUsageDescription",
|
||||
"Fluxer captures audio from the screen or window you choose to share.",
|
||||
),
|
||||
(
|
||||
"NSScreenCaptureUsageDescription",
|
||||
"Fluxer captures the screen or window you choose to share.",
|
||||
),
|
||||
];
|
||||
|
||||
pub fn install_desktop() -> Result<()> {
|
||||
run_command(
|
||||
&["pnpm", "install", "--frozen-lockfile"],
|
||||
RunOptions {
|
||||
cwd: ROOT.as_path(),
|
||||
env: PNPM_INSTALL_ENV
|
||||
.iter()
|
||||
.map(|(k, v)| ((*k).to_owned(), Some((*v).to_owned())))
|
||||
.collect(),
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
pub fn build_desktop(skip_native: bool) -> Result<()> {
|
||||
let env = vec![
|
||||
(
|
||||
"BUILD_CHANNEL".to_owned(),
|
||||
Some(env::var("BUILD_CHANNEL").unwrap_or_else(|_| "canary".to_owned())),
|
||||
),
|
||||
(
|
||||
"FLUXER_SKIP_NATIVE".to_owned(),
|
||||
if skip_native {
|
||||
Some("true".to_owned())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
),
|
||||
(
|
||||
"PUBLIC_BUILD_VERSION".to_owned(),
|
||||
Some(env::var("PUBLIC_BUILD_VERSION").unwrap_or_else(|_| "dev".to_owned())),
|
||||
),
|
||||
(
|
||||
"PUBLIC_RELEASE_CHANNEL".to_owned(),
|
||||
Some(env::var("PUBLIC_RELEASE_CHANNEL").unwrap_or_else(|_| "canary".to_owned())),
|
||||
),
|
||||
];
|
||||
run_command(
|
||||
&["pnpm", "build"],
|
||||
RunOptions {
|
||||
cwd: DESKTOP_DIR.as_path(),
|
||||
env,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
pub fn typecheck_desktop() -> Result<()> {
|
||||
run_command(
|
||||
&["pnpm", "typecheck"],
|
||||
RunOptions {
|
||||
cwd: DESKTOP_DIR.as_path(),
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
pub fn electron_args(args: &[String]) -> Vec<String> {
|
||||
let mut runtime_args = Vec::new();
|
||||
if cfg!(target_os = "linux")
|
||||
&& Path::new("/.dockerenv").exists()
|
||||
&& env::var("FLUXER_ELECTRON_NO_SANDBOX").as_deref() != Ok("0")
|
||||
{
|
||||
runtime_args.push("--no-sandbox".to_owned());
|
||||
}
|
||||
runtime_args.extend(args.iter().cloned());
|
||||
runtime_args
|
||||
}
|
||||
|
||||
pub fn electron_command(args: &[String], headless: bool) -> Vec<String> {
|
||||
let mut command = base_electron_command();
|
||||
command.extend(electron_args(args));
|
||||
if headless
|
||||
&& cfg!(target_os = "linux")
|
||||
&& env::var_os("DISPLAY").is_none()
|
||||
&& crate::paths::which("xvfb-run").is_some()
|
||||
{
|
||||
let mut wrapped = vec!["xvfb-run".to_owned(), "-a".to_owned()];
|
||||
wrapped.extend(command);
|
||||
return wrapped;
|
||||
}
|
||||
command
|
||||
}
|
||||
|
||||
fn base_electron_command() -> Vec<String> {
|
||||
if cfg!(target_os = "macos") && !Path::new("/.dockerenv").exists() {
|
||||
let electron_binary = dev_electron_binary_path();
|
||||
if electron_binary.is_file()
|
||||
&& let Ok(launcher) = env::current_exe()
|
||||
{
|
||||
return disclaimed_electron_command(&launcher, &electron_binary);
|
||||
}
|
||||
}
|
||||
vec![
|
||||
"pnpm".to_owned(),
|
||||
"exec".to_owned(),
|
||||
"electron".to_owned(),
|
||||
".".to_owned(),
|
||||
]
|
||||
}
|
||||
|
||||
fn disclaimed_electron_command(launcher: &Path, electron_binary: &Path) -> Vec<String> {
|
||||
vec![
|
||||
launcher.to_string_lossy().into_owned(),
|
||||
"desktop".to_owned(),
|
||||
"exec-disclaimed".to_owned(),
|
||||
electron_binary.to_string_lossy().into_owned(),
|
||||
".".to_owned(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn smoke_build_desktop() -> Result<()> {
|
||||
install_desktop()?;
|
||||
build_desktop(true)?;
|
||||
let command = electron_command(
|
||||
&[
|
||||
"--fluxer-debug-info".to_owned(),
|
||||
format!("--fluxer-app-url={LOCAL_APP_URL}"),
|
||||
],
|
||||
true,
|
||||
);
|
||||
let args: Vec<_> = command.iter().map(String::as_str).collect();
|
||||
run_command(
|
||||
&args,
|
||||
RunOptions {
|
||||
cwd: DESKTOP_DIR.as_path(),
|
||||
env: vec![("FLUXER_SKIP_NATIVE".to_owned(), Some("true".to_owned()))],
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
pub fn package_desktop(args: &[String]) -> Result<()> {
|
||||
build_desktop(false)?;
|
||||
let mut builder_args = vec![
|
||||
"pnpm".to_owned(),
|
||||
"exec".to_owned(),
|
||||
"electron-builder".to_owned(),
|
||||
"--dir".to_owned(),
|
||||
"--config".to_owned(),
|
||||
"electron-builder.config.cjs".to_owned(),
|
||||
];
|
||||
builder_args.extend(args.iter().cloned());
|
||||
let refs: Vec<_> = builder_args.iter().map(String::as_str).collect();
|
||||
run_command(
|
||||
&refs,
|
||||
RunOptions {
|
||||
cwd: DESKTOP_DIR.as_path(),
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
pub fn run_desktop(app_url: &str, extra_args: &[String], build: bool) -> Result<()> {
|
||||
install_desktop()?;
|
||||
if build {
|
||||
build_desktop(false)?;
|
||||
}
|
||||
run_desktop_process(app_url, extra_args)
|
||||
}
|
||||
|
||||
pub async fn run_desktop_canary(
|
||||
app_url: Option<&str>,
|
||||
extra_args: &[String],
|
||||
build: bool,
|
||||
) -> Result<()> {
|
||||
ensure_host_macos()?;
|
||||
let app_url = resolve_desktop_canary_app_url(app_url).await?;
|
||||
stop_running_canary_on_host()?;
|
||||
install_desktop()?;
|
||||
if build {
|
||||
build_desktop(false)?;
|
||||
}
|
||||
println!("Starting {CANARY_APP_NAME} against {app_url}");
|
||||
run_desktop_process(&app_url, extra_args)
|
||||
}
|
||||
|
||||
fn run_desktop_process(app_url: &str, extra_args: &[String]) -> Result<()> {
|
||||
patch_macos_dev_electron_info_plist()?;
|
||||
let mut args = vec![
|
||||
format!("--fluxer-app-url={app_url}"),
|
||||
"--fluxer-log-renderer-console".to_owned(),
|
||||
];
|
||||
args.extend(extra_args.iter().cloned());
|
||||
let command = electron_command(&args, false);
|
||||
let refs: Vec<_> = command.iter().map(String::as_str).collect();
|
||||
run_command(
|
||||
&refs,
|
||||
RunOptions {
|
||||
cwd: DESKTOP_DIR.as_path(),
|
||||
env: vec![
|
||||
("BUILD_CHANNEL".to_owned(), Some("canary".to_owned())),
|
||||
(
|
||||
"PUBLIC_BUILD_VERSION".to_owned(),
|
||||
Some(env::var("PUBLIC_BUILD_VERSION").unwrap_or_else(|_| "dev".to_owned())),
|
||||
),
|
||||
(
|
||||
"PUBLIC_RELEASE_CHANNEL".to_owned(),
|
||||
Some(
|
||||
env::var("PUBLIC_RELEASE_CHANNEL").unwrap_or_else(|_| "canary".to_owned()),
|
||||
),
|
||||
),
|
||||
],
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
fn patch_macos_dev_electron_info_plist() -> Result<()> {
|
||||
if !cfg!(target_os = "macos") || Path::new("/.dockerenv").exists() {
|
||||
return Ok(());
|
||||
}
|
||||
let app_bundle = dev_electron_app_bundle_path();
|
||||
let info_plist = dev_electron_info_plist_path();
|
||||
if !info_plist.is_file() {
|
||||
bail!(
|
||||
"missing dev Electron Info.plist at {}; run `pnpm dev:desktop:install` first",
|
||||
info_plist.display()
|
||||
);
|
||||
}
|
||||
|
||||
let mut changed = false;
|
||||
for (key, value) in MACOS_DEV_ELECTRON_USAGE_DESCRIPTIONS {
|
||||
changed |= set_or_add_plist_string(&info_plist, key, value)?;
|
||||
}
|
||||
|
||||
if changed {
|
||||
println!(
|
||||
"Patched dev Electron Info.plist for macOS capture permissions: {}",
|
||||
info_plist.display()
|
||||
);
|
||||
}
|
||||
if changed || !codesign_verify(&app_bundle) {
|
||||
codesign_ad_hoc(&app_bundle)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dev_electron_app_bundle_path() -> PathBuf {
|
||||
DESKTOP_DIR.join("node_modules/electron/dist/Electron.app")
|
||||
}
|
||||
|
||||
fn dev_electron_info_plist_path() -> PathBuf {
|
||||
dev_electron_app_bundle_path().join("Contents/Info.plist")
|
||||
}
|
||||
|
||||
fn dev_electron_binary_path() -> PathBuf {
|
||||
dev_electron_app_bundle_path().join("Contents/MacOS/Electron")
|
||||
}
|
||||
|
||||
fn set_or_add_plist_string(info_plist: &Path, key: &str, value: &str) -> Result<bool> {
|
||||
if read_plist_string(info_plist, key)?.as_deref() == Some(value) {
|
||||
return Ok(false);
|
||||
}
|
||||
if plist_key_exists(info_plist, key)? {
|
||||
run_plist_buddy(info_plist, &format!("Set :{key} {value}"))?;
|
||||
} else {
|
||||
run_plist_buddy(info_plist, &format!("Add :{key} string {value}"))?;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn plist_key_exists(info_plist: &Path, key: &str) -> Result<bool> {
|
||||
Ok(read_plist_string(info_plist, key)?.is_some())
|
||||
}
|
||||
|
||||
fn read_plist_string(info_plist: &Path, key: &str) -> Result<Option<String>> {
|
||||
let output = Command::new("/usr/libexec/PlistBuddy")
|
||||
.args(["-c", &format!("Print :{key}")])
|
||||
.arg(info_plist)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.with_context(|| format!("failed to read {key} from {}", info_plist.display()))?;
|
||||
if !output.status.success() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(
|
||||
String::from_utf8_lossy(&output.stdout)
|
||||
.trim_end()
|
||||
.to_owned(),
|
||||
))
|
||||
}
|
||||
|
||||
fn run_plist_buddy(info_plist: &Path, command: &str) -> Result<()> {
|
||||
let output = Command::new("/usr/libexec/PlistBuddy")
|
||||
.args(["-c", command])
|
||||
.arg(info_plist)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.with_context(|| format!("failed to update {}", info_plist.display()))?;
|
||||
if output.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
bail!(
|
||||
"failed to update {}: {}",
|
||||
info_plist.display(),
|
||||
String::from_utf8_lossy(&output.stderr).trim_end()
|
||||
)
|
||||
}
|
||||
|
||||
fn codesign_verify(app_bundle: &Path) -> bool {
|
||||
Command::new("codesign")
|
||||
.args(["--verify", "--deep", "--strict"])
|
||||
.arg(app_bundle)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status()
|
||||
.map(|status| status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn codesign_ad_hoc(app_bundle: &Path) -> Result<()> {
|
||||
println!("Re-signing dev Electron.app after macOS permission plist patch...");
|
||||
let output = Command::new("codesign")
|
||||
.args(["--force", "--deep", "--sign", "-"])
|
||||
.arg(app_bundle)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.with_context(|| format!("failed to re-sign {}", app_bundle.display()))?;
|
||||
if output.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
bail!(
|
||||
"failed to re-sign {}: {}",
|
||||
app_bundle.display(),
|
||||
String::from_utf8_lossy(&output.stderr).trim_end()
|
||||
)
|
||||
}
|
||||
|
||||
fn ensure_host_macos() -> Result<()> {
|
||||
if !cfg!(target_os = "macos") || Path::new("/.dockerenv").exists() {
|
||||
bail!(
|
||||
"`desktop canary` is a host macOS workflow. Run it on the Mac host, not inside the devcontainer."
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn resolve_desktop_canary_app_url(app_url: Option<&str>) -> Result<String> {
|
||||
if let Some(app_url) = app_url {
|
||||
return normalize_desktop_app_url(app_url);
|
||||
}
|
||||
for key in [
|
||||
"FLUXER_DESKTOP_CANARY_APP_URL",
|
||||
"FLUXER_DESKTOP_APP_URL",
|
||||
"FLUXER_PUBLIC_URL",
|
||||
] {
|
||||
if let Ok(value) = env::var(key) {
|
||||
let value = value.trim();
|
||||
if !value.is_empty()
|
||||
&& !is_local_app_url(value)
|
||||
&& public_app_url_is_reachable(value).await
|
||||
{
|
||||
return normalize_desktop_app_url(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(public_url) = crate::tunnel::resolve_cloudflare_public_url(None)
|
||||
&& public_app_url_is_reachable(&public_url).await
|
||||
{
|
||||
return normalize_desktop_app_url(&public_url);
|
||||
}
|
||||
Ok(LOCAL_APP_URL.to_owned())
|
||||
}
|
||||
|
||||
fn normalize_desktop_app_url(raw: &str) -> Result<String> {
|
||||
let url = Url::parse(raw.trim()).with_context(|| format!("invalid desktop app URL: {raw}"))?;
|
||||
match url.scheme() {
|
||||
"http" | "https" => Ok(url.as_str().trim_end_matches('/').to_owned()),
|
||||
scheme => bail!("desktop app URL must use http or https, got {scheme}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_local_app_url(raw: &str) -> bool {
|
||||
normalize_desktop_app_url(raw)
|
||||
.map(|url| matches!(url.as_str(), LOCAL_APP_URL | "http://127.0.0.1:8088"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn public_app_url_is_reachable(raw: &str) -> bool {
|
||||
let Ok(base_url) = normalize_desktop_app_url(raw) else {
|
||||
return false;
|
||||
};
|
||||
let client = match reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()
|
||||
{
|
||||
Ok(client) => client,
|
||||
Err(_) => return false,
|
||||
};
|
||||
client
|
||||
.get(format!("{base_url}/gateway/_health"))
|
||||
.send()
|
||||
.await
|
||||
.map(|response| response.status().is_success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn stop_running_canary_on_host() -> Result<()> {
|
||||
println!("Stopping any running {CANARY_APP_NAME} instance...");
|
||||
run_best_effort(
|
||||
"osascript",
|
||||
&[
|
||||
"-e",
|
||||
&format!("tell application id \"{CANARY_BUNDLE_ID}\" to quit"),
|
||||
],
|
||||
);
|
||||
run_best_effort(
|
||||
"osascript",
|
||||
&[
|
||||
"-e",
|
||||
&format!("tell application \"{CANARY_APP_NAME}\" to quit"),
|
||||
],
|
||||
);
|
||||
run_best_effort("pkill", &["-TERM", "-x", CANARY_APP_NAME]);
|
||||
terminate_rpc_port_processes(CANARY_RPC_PORT)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_best_effort(program: &str, args: &[&str]) {
|
||||
let _ = Command::new(program)
|
||||
.args(args)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.status();
|
||||
}
|
||||
|
||||
fn terminate_rpc_port_processes(port: u16) -> Result<()> {
|
||||
let mut pids = pids_listening_on_tcp_port(port)?;
|
||||
if pids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
kill_pids("-TERM", &pids)?;
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
while Instant::now() < deadline {
|
||||
thread::sleep(Duration::from_millis(250));
|
||||
pids = pids_listening_on_tcp_port(port)?;
|
||||
if pids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
kill_pids("-KILL", &pids)
|
||||
}
|
||||
|
||||
fn pids_listening_on_tcp_port(port: u16) -> Result<Vec<String>> {
|
||||
let output = Command::new("lsof")
|
||||
.args(["-ti", &format!("tcp:{port}")])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.output()
|
||||
.context("failed to run lsof while stopping Fluxer Canary")?;
|
||||
if !output.status.success() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout)
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn kill_pids(signal: &str, pids: &[String]) -> Result<()> {
|
||||
if pids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let status = Command::new("kill")
|
||||
.arg(signal)
|
||||
.args(pids)
|
||||
.status()
|
||||
.context("failed to run kill while stopping Fluxer Canary")?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
bail!(
|
||||
"failed to stop Fluxer Canary process(es): {}",
|
||||
pids.join(", ")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strips_no_sandbox_only_by_container_policy() {
|
||||
let args = electron_args(&["--flag".to_owned()]);
|
||||
if cfg!(target_os = "linux")
|
||||
&& Path::new("/.dockerenv").exists()
|
||||
&& env::var("FLUXER_ELECTRON_NO_SANDBOX").as_deref() != Ok("0")
|
||||
{
|
||||
assert_eq!(args[0], "--no-sandbox");
|
||||
}
|
||||
assert!(args.contains(&"--flag".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_desktop_app_urls() {
|
||||
assert_eq!(
|
||||
normalize_desktop_app_url("https://dev.example.test/").unwrap(),
|
||||
"https://dev.example.test"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_desktop_app_url("http://localhost:8088").unwrap(),
|
||||
LOCAL_APP_URL
|
||||
);
|
||||
assert!(normalize_desktop_app_url("wss://dev.example.test").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disclaimed_electron_command_re_execs_through_the_launcher() {
|
||||
let command = disclaimed_electron_command(
|
||||
Path::new("/tmp/fluxer-dev"),
|
||||
Path::new("/tmp/Electron.app/Contents/MacOS/Electron"),
|
||||
);
|
||||
assert_eq!(
|
||||
command,
|
||||
vec![
|
||||
"/tmp/fluxer-dev",
|
||||
"desktop",
|
||||
"exec-disclaimed",
|
||||
"/tmp/Electron.app/Contents/MacOS/Electron",
|
||||
"."
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dev_electron_binary_path_points_inside_the_dev_bundle() {
|
||||
assert!(dev_electron_binary_path().ends_with(
|
||||
"fluxer_desktop/node_modules/electron/dist/Electron.app/Contents/MacOS/Electron"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dev_electron_plist_path_points_to_fluxer_desktop_electron_app() {
|
||||
assert!(dev_electron_info_plist_path().ends_with(
|
||||
"fluxer_desktop/node_modules/electron/dist/Electron.app/Contents/Info.plist"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dev_electron_usage_descriptions_include_audio_capture() {
|
||||
assert!(
|
||||
MACOS_DEV_ELECTRON_USAGE_DESCRIPTIONS
|
||||
.iter()
|
||||
.any(|(key, _)| *key == "NSAudioCaptureUsageDescription")
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,81 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::ffi::CString;
|
||||
|
||||
pub fn exec_disclaimed(program: &str, args: &[String]) -> Result<()> {
|
||||
let program_c = CString::new(program).context("program path contains a NUL byte")?;
|
||||
let mut argv = Vec::with_capacity(args.len() + 1);
|
||||
argv.push(program_c.clone());
|
||||
for arg in args {
|
||||
argv.push(CString::new(arg.as_str()).context("argument contains a NUL byte")?);
|
||||
}
|
||||
exec_disclaimed_impl(&program_c, &argv)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn exec_disclaimed_impl(program: &CString, argv: &[CString]) -> Result<()> {
|
||||
use std::ptr;
|
||||
|
||||
unsafe extern "C" {
|
||||
fn responsibility_spawnattrs_setdisclaim(
|
||||
attrs: *mut libc::posix_spawnattr_t,
|
||||
disclaim: libc::c_int,
|
||||
) -> libc::c_int;
|
||||
}
|
||||
|
||||
let mut argv_ptrs: Vec<*mut libc::c_char> =
|
||||
argv.iter().map(|arg| arg.as_ptr().cast_mut()).collect();
|
||||
argv_ptrs.push(ptr::null_mut());
|
||||
|
||||
unsafe {
|
||||
let mut attrs: libc::posix_spawnattr_t = ptr::null_mut();
|
||||
let rc = libc::posix_spawnattr_init(&mut attrs);
|
||||
if rc != 0 {
|
||||
bail!(
|
||||
"posix_spawnattr_init failed: {}",
|
||||
std::io::Error::from_raw_os_error(rc)
|
||||
);
|
||||
}
|
||||
let rc =
|
||||
libc::posix_spawnattr_setflags(&mut attrs, libc::POSIX_SPAWN_SETEXEC as libc::c_short);
|
||||
if rc != 0 {
|
||||
libc::posix_spawnattr_destroy(&mut attrs);
|
||||
bail!(
|
||||
"posix_spawnattr_setflags failed: {}",
|
||||
std::io::Error::from_raw_os_error(rc)
|
||||
);
|
||||
}
|
||||
let rc = responsibility_spawnattrs_setdisclaim(&mut attrs, 1);
|
||||
if rc != 0 {
|
||||
libc::posix_spawnattr_destroy(&mut attrs);
|
||||
bail!(
|
||||
"responsibility_spawnattrs_setdisclaim failed: {}",
|
||||
std::io::Error::from_raw_os_error(rc)
|
||||
);
|
||||
}
|
||||
let mut pid: libc::pid_t = 0;
|
||||
let rc = libc::posix_spawn(
|
||||
&mut pid,
|
||||
program.as_ptr(),
|
||||
ptr::null(),
|
||||
&attrs,
|
||||
argv_ptrs.as_ptr(),
|
||||
*libc::_NSGetEnviron(),
|
||||
);
|
||||
libc::posix_spawnattr_destroy(&mut attrs);
|
||||
bail!(
|
||||
"posix_spawn(SETEXEC) failed for {}: {}",
|
||||
program.to_string_lossy(),
|
||||
std::io::Error::from_raw_os_error(rc)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
fn exec_disclaimed_impl(program: &CString, _argv: &[CString]) -> Result<()> {
|
||||
bail!(
|
||||
"exec-disclaimed is only supported on macOS (requested program: {})",
|
||||
program.to_string_lossy()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn read_env_file(path: &Path) -> Result<BTreeMap<String, String>> {
|
||||
let mut values = BTreeMap::new();
|
||||
if !path.exists() {
|
||||
return Ok(values);
|
||||
}
|
||||
let text = std::fs::read_to_string(path)
|
||||
.with_context(|| format!("failed to read {}", path.display()))?;
|
||||
for (index, raw_line) in text.lines().enumerate() {
|
||||
let line_number = index + 1;
|
||||
let mut line = raw_line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if let Some(rest) = line.strip_prefix("export ") {
|
||||
line = rest.trim();
|
||||
}
|
||||
let Some((key, value)) = line.split_once('=') else {
|
||||
bail!(
|
||||
"Invalid env line in {}:{}: {}",
|
||||
path.display(),
|
||||
line_number,
|
||||
raw_line
|
||||
);
|
||||
};
|
||||
let key = key.trim();
|
||||
if key.is_empty() {
|
||||
bail!("Missing env key in {}:{line_number}", path.display());
|
||||
}
|
||||
values.insert(key.to_owned(), parse_env_value(value.trim()));
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
pub fn read_env_files(paths: &[&Path]) -> Result<BTreeMap<String, String>> {
|
||||
let mut values = BTreeMap::new();
|
||||
for path in paths {
|
||||
values.extend(read_env_file(path)?);
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
pub fn merge_env_layers(layers: &[BTreeMap<String, String>]) -> BTreeMap<String, String> {
|
||||
let mut merged = BTreeMap::new();
|
||||
for layer in layers {
|
||||
merged.extend(layer.clone());
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
pub fn merge_default_env_with_current(
|
||||
development_path: &Path,
|
||||
local_path: &Path,
|
||||
root_local_path: &Path,
|
||||
current: BTreeMap<String, String>,
|
||||
) -> Result<BTreeMap<String, String>> {
|
||||
merge_default_env_with_current_and_baseline(
|
||||
development_path,
|
||||
local_path,
|
||||
root_local_path,
|
||||
current,
|
||||
read_container_initial_env().unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
fn merge_default_env_with_current_and_baseline(
|
||||
development_path: &Path,
|
||||
local_path: &Path,
|
||||
root_local_path: &Path,
|
||||
current: BTreeMap<String, String>,
|
||||
baseline: BTreeMap<String, String>,
|
||||
) -> Result<BTreeMap<String, String>> {
|
||||
let development = read_env_file(development_path)?;
|
||||
let local = read_env_file(local_path)?;
|
||||
let root_local = read_env_file(root_local_path)?;
|
||||
let mut effective_current = current;
|
||||
for (key, value) in &development {
|
||||
let has_file_override = local.contains_key(key) || root_local.contains_key(key);
|
||||
let current_is_development_default = effective_current.get(key) == Some(value);
|
||||
let current_is_container_default = baseline
|
||||
.get(key)
|
||||
.is_some_and(|baseline_value| effective_current.get(key) == Some(baseline_value));
|
||||
if has_file_override && (current_is_development_default || current_is_container_default) {
|
||||
effective_current.remove(key);
|
||||
}
|
||||
}
|
||||
Ok(merge_env_layers(&[
|
||||
development,
|
||||
local,
|
||||
root_local,
|
||||
effective_current,
|
||||
]))
|
||||
}
|
||||
|
||||
fn read_container_initial_env() -> Option<BTreeMap<String, String>> {
|
||||
let bytes = std::fs::read("/proc/1/environ").ok()?;
|
||||
let mut values = BTreeMap::new();
|
||||
for entry in bytes.split(|byte| *byte == 0) {
|
||||
if entry.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let text = String::from_utf8_lossy(entry);
|
||||
let Some((key, value)) = text.split_once('=') else {
|
||||
continue;
|
||||
};
|
||||
values.insert(key.to_owned(), value.to_owned());
|
||||
}
|
||||
Some(values)
|
||||
}
|
||||
|
||||
fn parse_env_value(mut value: &str) -> String {
|
||||
if value.len() >= 2 {
|
||||
let bytes = value.as_bytes();
|
||||
if (bytes[0] == b'\'' || bytes[0] == b'"') && bytes[0] == bytes[value.len() - 1] {
|
||||
return value[1..value.len() - 1].to_owned();
|
||||
}
|
||||
}
|
||||
if let Some((before, _)) = value.split_once(" #") {
|
||||
value = before.trim_end();
|
||||
}
|
||||
value.to_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_env_values_with_shell_like_comments() {
|
||||
assert_eq!(parse_env_value("'quoted'"), "quoted");
|
||||
assert_eq!(parse_env_value("\"quoted\""), "quoted");
|
||||
assert_eq!(parse_env_value("value # comment"), "value");
|
||||
assert_eq!(parse_env_value("value#not-comment"), "value#not-comment");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_exports_and_reports_invalid_lines() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let file = dir.path().join("env");
|
||||
std::fs::write(
|
||||
&file,
|
||||
"\n# comment\nexport A=1\nB='two words'\nC=value # trailing\n",
|
||||
)
|
||||
.unwrap();
|
||||
let values = read_env_file(&file).unwrap();
|
||||
assert_eq!(values.get("A").unwrap(), "1");
|
||||
assert_eq!(values.get("B").unwrap(), "two words");
|
||||
assert_eq!(values.get("C").unwrap(), "value");
|
||||
|
||||
std::fs::write(&file, "nope\n").unwrap();
|
||||
assert!(
|
||||
read_env_file(&file)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("Invalid env line")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_env_overrides_injected_development_defaults() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let development = dir.path().join("development.env");
|
||||
let local = dir.path().join("local.env");
|
||||
let root_local = dir.path().join(".env.local");
|
||||
std::fs::write(&development, "A=default\nB=default\nC=default\n").unwrap();
|
||||
std::fs::write(&local, "A=local\n").unwrap();
|
||||
std::fs::write(&root_local, "B=root\n").unwrap();
|
||||
let current = BTreeMap::from([
|
||||
("A".to_owned(), "default".to_owned()),
|
||||
("B".to_owned(), "custom".to_owned()),
|
||||
("C".to_owned(), "default".to_owned()),
|
||||
]);
|
||||
let merged = merge_default_env_with_current_and_baseline(
|
||||
&development,
|
||||
&local,
|
||||
&root_local,
|
||||
current,
|
||||
BTreeMap::new(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(merged.get("A").map(String::as_str), Some("local"));
|
||||
assert_eq!(merged.get("B").map(String::as_str), Some("custom"));
|
||||
assert_eq!(merged.get("C").map(String::as_str), Some("default"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_container_defaults_yield_to_file_overrides() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let development = dir.path().join("development.env");
|
||||
let local = dir.path().join("local.env");
|
||||
let root_local = dir.path().join(".env.local");
|
||||
std::fs::write(
|
||||
&development,
|
||||
"A=new-default\nB=new-default\nC=new-default\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(&local, "A=local\nC=local\n").unwrap();
|
||||
std::fs::write(&root_local, "").unwrap();
|
||||
let baseline = BTreeMap::from([
|
||||
("A".to_owned(), "old-default".to_owned()),
|
||||
("B".to_owned(), "old-default".to_owned()),
|
||||
("C".to_owned(), "old-default".to_owned()),
|
||||
]);
|
||||
let current = BTreeMap::from([
|
||||
("A".to_owned(), "old-default".to_owned()),
|
||||
("B".to_owned(), "custom".to_owned()),
|
||||
("C".to_owned(), "old-default".to_owned()),
|
||||
]);
|
||||
let merged = merge_default_env_with_current_and_baseline(
|
||||
&development,
|
||||
&local,
|
||||
&root_local,
|
||||
current,
|
||||
baseline,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(merged.get("A").map(String::as_str), Some("local"));
|
||||
assert_eq!(merged.get("B").map(String::as_str), Some("custom"));
|
||||
assert_eq!(merged.get("C").map(String::as_str), Some("local"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,420 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::gateway::{GatewayNode, gateway_dir};
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::mpsc::{Receiver, channel};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
const WATCH_POLL_INTERVAL: Duration = Duration::from_millis(1000);
|
||||
const WATCH_SETTLE_INTERVAL: Duration = Duration::from_millis(300);
|
||||
const RELOAD_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const WATCHED_SOURCE_EXTENSIONS: &[&str] = &["erl", "hrl", "src", "rs", "toml", "config"];
|
||||
|
||||
pub type FileState = BTreeMap<PathBuf, (SystemTime, u64)>;
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ArtifactState {
|
||||
pub beams: FileState,
|
||||
pub nifs: FileState,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ArtifactDiff {
|
||||
pub modules: Vec<String>,
|
||||
pub nifs_changed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ReloadOutcome {
|
||||
pub failed_nodes: Vec<String>,
|
||||
}
|
||||
|
||||
pub fn hot_reload_enabled() -> bool {
|
||||
env::var("FLUXER_DEV_GATEWAY_HOT_RELOAD")
|
||||
.map(|value| !matches!(value.to_ascii_lowercase().as_str(), "0" | "false" | "no"))
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
pub fn spawn_source_watcher() -> Receiver<()> {
|
||||
let (sender, receiver) = channel();
|
||||
std::thread::spawn(move || {
|
||||
let mut previous = scan_sources();
|
||||
loop {
|
||||
std::thread::sleep(WATCH_POLL_INTERVAL);
|
||||
let mut current = scan_sources();
|
||||
if current == previous {
|
||||
continue;
|
||||
}
|
||||
loop {
|
||||
std::thread::sleep(WATCH_SETTLE_INTERVAL);
|
||||
let settled = scan_sources();
|
||||
if settled == current {
|
||||
break;
|
||||
}
|
||||
current = settled;
|
||||
}
|
||||
previous = current;
|
||||
if sender.send(()).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
receiver
|
||||
}
|
||||
|
||||
fn scan_sources() -> FileState {
|
||||
let dir = gateway_dir();
|
||||
let mut state = FileState::new();
|
||||
for root in [dir.join("src"), dir.join("include"), dir.join("native")] {
|
||||
scan_tree(&mut state, &root);
|
||||
}
|
||||
for path in [
|
||||
dir.join("rebar.config"),
|
||||
dir.join("rebar.config.script"),
|
||||
dir.join("rebar.lock"),
|
||||
] {
|
||||
record_file(&mut state, &path);
|
||||
}
|
||||
state
|
||||
}
|
||||
|
||||
fn scan_tree(state: &mut FileState, root: &Path) {
|
||||
let Ok(entries) = fs::read_dir(root) else {
|
||||
return;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let Ok(metadata) = entry.metadata() else {
|
||||
continue;
|
||||
};
|
||||
if metadata.is_dir() {
|
||||
if !is_skipped_dir(&path) {
|
||||
scan_tree(state, &path);
|
||||
}
|
||||
} else if is_watched_source(&path)
|
||||
&& let Ok(modified) = metadata.modified()
|
||||
{
|
||||
state.insert(path, (modified, metadata.len()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_file(state: &mut FileState, path: &Path) {
|
||||
if let Ok(metadata) = fs::metadata(path)
|
||||
&& let Ok(modified) = metadata.modified()
|
||||
{
|
||||
state.insert(path.to_path_buf(), (modified, metadata.len()));
|
||||
}
|
||||
}
|
||||
|
||||
fn is_skipped_dir(path: &Path) -> bool {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name == "target" || name.starts_with('.'))
|
||||
}
|
||||
|
||||
fn is_watched_source(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|extension| extension.to_str())
|
||||
.is_some_and(|extension| WATCHED_SOURCE_EXTENSIONS.contains(&extension))
|
||||
}
|
||||
|
||||
pub fn snapshot_artifacts() -> ArtifactState {
|
||||
let mut state = ArtifactState::default();
|
||||
let lib_root = gateway_dir().join("_build/default/lib");
|
||||
if let Ok(entries) = fs::read_dir(&lib_root) {
|
||||
for entry in entries.flatten() {
|
||||
scan_artifact_dir(&mut state.beams, &entry.path().join("ebin"), "beam");
|
||||
}
|
||||
}
|
||||
scan_artifact_dir(&mut state.nifs, &gateway_dir().join("priv"), "so");
|
||||
state
|
||||
}
|
||||
|
||||
fn scan_artifact_dir(state: &mut FileState, dir: &Path, extension: &str) {
|
||||
let Ok(entries) = fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path
|
||||
.extension()
|
||||
.is_some_and(|candidate| candidate == extension)
|
||||
&& let Ok(metadata) = entry.metadata()
|
||||
&& let Ok(modified) = metadata.modified()
|
||||
{
|
||||
state.insert(path, (modified, metadata.len()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn changed_artifacts(before: &ArtifactState, after: &ArtifactState) -> ArtifactDiff {
|
||||
let modules = after
|
||||
.beams
|
||||
.iter()
|
||||
.filter(|(path, state)| before.beams.get(*path) != Some(*state))
|
||||
.filter_map(|(path, _)| Some(path.file_stem()?.to_str()?.to_owned()))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let nifs_changed = after
|
||||
.nifs
|
||||
.iter()
|
||||
.any(|(path, state)| before.nifs.get(path) != Some(state));
|
||||
ArtifactDiff {
|
||||
modules,
|
||||
nifs_changed,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_reload_eval(nodes: &[String], modules: &[String]) -> String {
|
||||
let node_list = nodes
|
||||
.iter()
|
||||
.map(|node| format!("'{node}'"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let module_list = modules
|
||||
.iter()
|
||||
.map(|module| format!("'{module}'"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(
|
||||
"Nodes = [{node_list}], \
|
||||
Mods = [{module_list}], \
|
||||
Failed = [Node || Node <- Nodes, \
|
||||
case net_adm:ping(Node) of \
|
||||
pong -> \
|
||||
Errors = [Mod || Mod <- Mods, \
|
||||
begin \
|
||||
rpc:call(Node, code, purge, [Mod], 10000), \
|
||||
case rpc:call(Node, code, load_file, [Mod], 10000) of \
|
||||
{{module, Mod}} -> false; \
|
||||
Other -> \
|
||||
io:format(\"gateway_reload_error ~s ~s ~0p~n\", [Node, Mod, Other]), \
|
||||
true \
|
||||
end \
|
||||
end], \
|
||||
Errors =/= []; \
|
||||
pang -> \
|
||||
io:format(\"gateway_reload_node_down ~s~n\", [Node]), \
|
||||
true \
|
||||
end], \
|
||||
[io:format(\"gateway_reload_failed_node ~s~n\", [Failed1]) || Failed1 <- Failed], \
|
||||
io:format(\"gateway_reload_done~n\"), \
|
||||
halt(0)."
|
||||
)
|
||||
}
|
||||
|
||||
pub fn hot_reload_modules(
|
||||
nodes: &[GatewayNode],
|
||||
modules: &[String],
|
||||
cookie: &str,
|
||||
) -> Result<ReloadOutcome> {
|
||||
assert!(!modules.is_empty());
|
||||
let node_names = nodes
|
||||
.iter()
|
||||
.map(GatewayNode::erlang_name)
|
||||
.collect::<Vec<_>>();
|
||||
let eval = build_reload_eval(&node_names, modules);
|
||||
let reloader_name = format!("fluxer_dev_reload_{}@127.0.0.1", std::process::id());
|
||||
let mut child = Command::new("erl")
|
||||
.args([
|
||||
"-hidden",
|
||||
"-noshell",
|
||||
"-name",
|
||||
&reloader_name,
|
||||
"-setcookie",
|
||||
cookie,
|
||||
"-eval",
|
||||
&eval,
|
||||
])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("failed to spawn gateway reload shell")?;
|
||||
let stdout = child.stdout.take().map(spawn_reader);
|
||||
let stderr = child.stderr.take().map(spawn_reader);
|
||||
let status = wait_with_deadline(&mut child, RELOAD_TIMEOUT)?;
|
||||
let mut output = String::new();
|
||||
for handle in [stdout, stderr].into_iter().flatten() {
|
||||
if let Ok(text) = handle.join() {
|
||||
output.push_str(&text);
|
||||
}
|
||||
}
|
||||
for line in output.lines().filter(|line| !line.trim().is_empty()) {
|
||||
println!("[gateway:reload] {line}");
|
||||
}
|
||||
if !status.success() {
|
||||
bail!(
|
||||
"gateway reload shell exited with status {}",
|
||||
status.code().unwrap_or(1)
|
||||
);
|
||||
}
|
||||
if !output.lines().any(|line| line == "gateway_reload_done") {
|
||||
bail!("gateway reload shell did not report completion");
|
||||
}
|
||||
Ok(parse_reload_outcome(&output))
|
||||
}
|
||||
|
||||
pub fn parse_reload_outcome(output: &str) -> ReloadOutcome {
|
||||
let failed_nodes = output
|
||||
.lines()
|
||||
.filter_map(|line| line.strip_prefix("gateway_reload_failed_node "))
|
||||
.map(|node| node.trim().to_owned())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
ReloadOutcome { failed_nodes }
|
||||
}
|
||||
|
||||
fn spawn_reader(stream: impl Read + Send + 'static) -> std::thread::JoinHandle<String> {
|
||||
std::thread::spawn(move || {
|
||||
let mut reader = stream;
|
||||
let mut text = String::new();
|
||||
let mut bytes = Vec::new();
|
||||
if reader.read_to_end(&mut bytes).is_ok() {
|
||||
text = String::from_utf8_lossy(&bytes).into_owned();
|
||||
}
|
||||
text
|
||||
})
|
||||
}
|
||||
|
||||
fn wait_with_deadline(child: &mut Child, timeout: Duration) -> Result<std::process::ExitStatus> {
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
if let Some(status) = child.try_wait()? {
|
||||
return Ok(status);
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
bail!(
|
||||
"gateway reload shell timed out after {}s",
|
||||
timeout.as_secs()
|
||||
);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn file_state(entries: &[(&str, u64, u64)]) -> FileState {
|
||||
entries
|
||||
.iter()
|
||||
.map(|(path, seconds, size)| {
|
||||
(
|
||||
PathBuf::from(path),
|
||||
(
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(*seconds),
|
||||
*size,
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_artifacts_detects_new_and_modified_beams() {
|
||||
let before = ArtifactState {
|
||||
beams: file_state(&[("ebin/a.beam", 1, 10), ("ebin/b.beam", 1, 20)]),
|
||||
nifs: FileState::new(),
|
||||
};
|
||||
let after = ArtifactState {
|
||||
beams: file_state(&[
|
||||
("ebin/a.beam", 2, 10),
|
||||
("ebin/b.beam", 1, 20),
|
||||
("ebin/c.beam", 1, 30),
|
||||
]),
|
||||
nifs: FileState::new(),
|
||||
};
|
||||
let diff = changed_artifacts(&before, &after);
|
||||
assert_eq!(diff.modules, vec!["a".to_owned(), "c".to_owned()]);
|
||||
assert!(!diff.nifs_changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_artifacts_ignores_unchanged_state() {
|
||||
let state = ArtifactState {
|
||||
beams: file_state(&[("ebin/a.beam", 1, 10)]),
|
||||
nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
|
||||
};
|
||||
let diff = changed_artifacts(&state, &state.clone());
|
||||
assert!(diff.modules.is_empty());
|
||||
assert!(!diff.nifs_changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_artifacts_flags_nif_changes() {
|
||||
let before = ArtifactState {
|
||||
beams: FileState::new(),
|
||||
nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
|
||||
};
|
||||
let after = ArtifactState {
|
||||
beams: FileState::new(),
|
||||
nifs: file_state(&[("priv/a_nif.so", 2, 11)]),
|
||||
};
|
||||
assert!(changed_artifacts(&before, &after).nifs_changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_eval_quotes_nodes_and_modules() {
|
||||
let eval = build_reload_eval(
|
||||
&["fluxer_gateway_websocket_1@127.0.0.1".to_owned()],
|
||||
&["gateway_compress".to_owned(), "push".to_owned()],
|
||||
);
|
||||
assert!(eval.contains("Nodes = ['fluxer_gateway_websocket_1@127.0.0.1']"));
|
||||
assert!(eval.contains("Mods = ['gateway_compress','push']"));
|
||||
assert!(eval.contains("halt(0)."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_outcome_parses_failed_nodes() {
|
||||
let output = "gateway_reload_error n1 mod {error,nofile}\n\
|
||||
gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\
|
||||
gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\
|
||||
gateway_reload_done\n";
|
||||
let outcome = parse_reload_outcome(output);
|
||||
assert_eq!(
|
||||
outcome.failed_nodes,
|
||||
vec!["fluxer_gateway_guilds_2@127.0.0.1".to_owned()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_outcome_empty_when_no_failures() {
|
||||
assert!(
|
||||
parse_reload_outcome("gateway_reload_done\n")
|
||||
.failed_nodes
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watched_source_filter_accepts_known_extensions() {
|
||||
assert!(is_watched_source(Path::new("src/push.erl")));
|
||||
assert!(is_watched_source(Path::new("include/gateway.hrl")));
|
||||
assert!(is_watched_source(Path::new("src/fluxer_gateway.app.src")));
|
||||
assert!(is_watched_source(Path::new("native/a_nif/src/lib.rs")));
|
||||
assert!(is_watched_source(Path::new("native/a_nif/Cargo.toml")));
|
||||
assert!(!is_watched_source(Path::new("ebin/push.beam")));
|
||||
assert!(!is_watched_source(Path::new("src/.push.erl.swp")));
|
||||
assert!(!is_watched_source(Path::new("native/a_nif/Cargo.lock")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skipped_dirs_cover_build_output_and_hidden_dirs() {
|
||||
assert!(is_skipped_dir(Path::new("native/a_nif/target")));
|
||||
assert!(is_skipped_dir(Path::new("src/.git")));
|
||||
assert!(!is_skipped_dir(Path::new("src/gateway")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
pub mod bootstrap;
|
||||
pub mod cassandra;
|
||||
pub mod desktop;
|
||||
pub mod dev;
|
||||
pub mod disclaim;
|
||||
pub mod env;
|
||||
pub mod gateway;
|
||||
pub mod gateway_reload;
|
||||
pub mod local_k8s;
|
||||
pub mod manifest;
|
||||
pub mod marketing;
|
||||
pub mod media_proxy;
|
||||
pub mod media_stress;
|
||||
pub mod native_voice_it;
|
||||
pub mod paths;
|
||||
pub mod proc;
|
||||
pub mod proxy;
|
||||
pub mod rust_services;
|
||||
pub mod smoke;
|
||||
pub mod tasks;
|
||||
pub mod tunnel;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,413 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::{Args, Parser, Subcommand};
|
||||
use fluxer_dev::cassandra::{
|
||||
apply_schema, compute_diff, render_target_schema, verify_schema, write_diff_file,
|
||||
};
|
||||
use fluxer_dev::desktop::{
|
||||
build_desktop, install_desktop, package_desktop, run_desktop, run_desktop_canary,
|
||||
smoke_build_desktop, typecheck_desktop,
|
||||
};
|
||||
use fluxer_dev::env::merge_default_env_with_current;
|
||||
use fluxer_dev::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL};
|
||||
use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT_LOCAL_ENV_FILE};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "fluxer-dev")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Command {
|
||||
Bootstrap(BootstrapArgs),
|
||||
PostStart,
|
||||
Gateway(GatewayArgs),
|
||||
Build,
|
||||
Knip,
|
||||
Test,
|
||||
Typecheck,
|
||||
Proxy(ProxyArgs),
|
||||
Dev(DevArgs),
|
||||
RustServices(RustServicesArgs),
|
||||
Smoke(SmokeArgs),
|
||||
Cassandra(CassandraArgs),
|
||||
Desktop(DesktopArgs),
|
||||
LocalK8s(LocalK8sArgs),
|
||||
Marketing(MarketingArgs),
|
||||
MediaProxy(MediaProxyArgs),
|
||||
Tunnel(TunnelArgs),
|
||||
NativeVoiceIt(fluxer_dev::native_voice_it::NativeVoiceItArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct BootstrapArgs {
|
||||
#[arg(long)]
|
||||
skip_install: bool,
|
||||
#[arg(long)]
|
||||
skip_desktop_install: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct GatewayArgs {
|
||||
#[arg(value_parser = ["cluster", "single"], default_value = "cluster")]
|
||||
mode: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct ProxyArgs {
|
||||
#[arg(long, default_value = "0.0.0.0")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = DEV_PROXY_PORT)]
|
||||
port: u16,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct DevArgs {
|
||||
#[arg(long)]
|
||||
cloudflare_tunnel: bool,
|
||||
#[arg(long)]
|
||||
public_url: Option<String>,
|
||||
tasks: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct RustServicesArgs {
|
||||
services: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct SmokeArgs {
|
||||
#[arg(long)]
|
||||
quick: bool,
|
||||
#[arg(long)]
|
||||
public: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct CassandraArgs {
|
||||
#[command(subcommand)]
|
||||
command: CassandraCommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum CassandraCommand {
|
||||
Diff {
|
||||
#[arg(long)]
|
||||
output: Option<PathBuf>,
|
||||
},
|
||||
Apply,
|
||||
Verify,
|
||||
TargetSchema,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct DesktopArgs {
|
||||
#[command(subcommand)]
|
||||
command: DesktopCommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum DesktopCommand {
|
||||
Install,
|
||||
Build {
|
||||
#[arg(long)]
|
||||
skip_native: bool,
|
||||
},
|
||||
Typecheck,
|
||||
SmokeBuild,
|
||||
Package {
|
||||
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
|
||||
builder_args: Vec<String>,
|
||||
},
|
||||
Run {
|
||||
#[arg(long, default_value = LOCAL_APP_URL)]
|
||||
app_url: String,
|
||||
#[arg(long)]
|
||||
no_build: bool,
|
||||
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
|
||||
extra_args: Vec<String>,
|
||||
},
|
||||
Canary {
|
||||
#[arg(long)]
|
||||
app_url: Option<String>,
|
||||
#[arg(long)]
|
||||
no_build: bool,
|
||||
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
|
||||
extra_args: Vec<String>,
|
||||
},
|
||||
#[command(hide = true)]
|
||||
ExecDisclaimed {
|
||||
program: String,
|
||||
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
|
||||
args: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct LocalK8sArgs {
|
||||
#[command(subcommand)]
|
||||
command: LocalK8sCommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum LocalK8sCommand {
|
||||
CreateCluster,
|
||||
Kubectl {
|
||||
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
|
||||
args: Vec<String>,
|
||||
},
|
||||
Helm {
|
||||
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
|
||||
args: Vec<String>,
|
||||
},
|
||||
HotpatchSmoke,
|
||||
HandoffRolloutSmoke,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct MarketingArgs {
|
||||
#[command(subcommand)]
|
||||
command: MarketingCommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum MarketingCommand {
|
||||
PreprocessBlogImage(fluxer_dev::marketing::PreprocessBlogImageArgs),
|
||||
PreprocessBlogVideo(fluxer_dev::marketing::PreprocessBlogVideoArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct MediaProxyArgs {
|
||||
#[command(subcommand)]
|
||||
command: MediaProxyCommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum MediaProxyCommand {
|
||||
BeeGifBench,
|
||||
Doctor {
|
||||
#[arg(long)]
|
||||
repair: bool,
|
||||
#[arg(long, default_value = LOCAL_APP_URL)]
|
||||
base_url: String,
|
||||
#[arg(long)]
|
||||
path: Option<String>,
|
||||
},
|
||||
SeaweedfsIntegration {
|
||||
#[arg(long)]
|
||||
isolated_store: bool,
|
||||
},
|
||||
RustStressSmoke,
|
||||
StressCompare(fluxer_dev::media_stress::StressCompareArgs),
|
||||
SignExternalUrl(fluxer_dev::media_stress::SignExternalUrlArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct TunnelArgs {
|
||||
#[command(subcommand)]
|
||||
command: TunnelCommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum TunnelCommand {
|
||||
Configure {
|
||||
#[arg(long)]
|
||||
public_url: String,
|
||||
#[arg(long, hide_env_values = true)]
|
||||
token: Option<String>,
|
||||
},
|
||||
PrintEnv {
|
||||
#[arg(long)]
|
||||
public_url: String,
|
||||
},
|
||||
Run {
|
||||
#[arg(long, env = "FLUXER_CLOUDFLARE_TUNNEL_TOKEN", hide_env_values = true)]
|
||||
token: Option<String>,
|
||||
#[arg(long)]
|
||||
token_file: Option<PathBuf>,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
if std::env::args_os()
|
||||
.next()
|
||||
.and_then(|path| {
|
||||
std::path::PathBuf::from(path)
|
||||
.file_name()
|
||||
.map(|name| name.to_owned())
|
||||
})
|
||||
.as_deref()
|
||||
== Some(std::ffi::OsStr::new("docker"))
|
||||
{
|
||||
std::process::exit(fluxer_dev::local_k8s::run_docker_wrapper(
|
||||
std::env::args_os().skip(1),
|
||||
));
|
||||
}
|
||||
|
||||
let cli = Cli::parse();
|
||||
if !matches!(
|
||||
cli.command,
|
||||
Command::Build | Command::Knip | Command::Test | Command::Typecheck
|
||||
) {
|
||||
apply_default_env()?;
|
||||
}
|
||||
|
||||
match cli.command {
|
||||
Command::Bootstrap(args) => {
|
||||
fluxer_dev::bootstrap::bootstrap(args.skip_install, args.skip_desktop_install).await?;
|
||||
}
|
||||
Command::PostStart => fluxer_dev::bootstrap::post_start().await?,
|
||||
Command::Gateway(args) if args.mode == "single" => fluxer_dev::gateway::run_gateway()?,
|
||||
Command::Gateway(_) => {
|
||||
std::process::exit(fluxer_dev::gateway::run_gateway_cluster().await?)
|
||||
}
|
||||
Command::Build => std::process::exit(fluxer_dev::tasks::run_build()?),
|
||||
Command::Knip => std::process::exit(fluxer_dev::tasks::run_knip()?),
|
||||
Command::Test => std::process::exit(fluxer_dev::tasks::run_test()?),
|
||||
Command::Typecheck => std::process::exit(fluxer_dev::tasks::run_typecheck()?),
|
||||
Command::Proxy(args) => fluxer_dev::proxy::run_proxy(&args.host, args.port).await?,
|
||||
Command::Dev(args) => {
|
||||
if args.cloudflare_tunnel {
|
||||
fluxer_dev::tunnel::apply_cloudflare_public_url_env(args.public_url.as_deref())?;
|
||||
} else if let Some(public_url) = args.public_url.as_deref() {
|
||||
fluxer_dev::tunnel::apply_public_url_env(public_url)?;
|
||||
}
|
||||
std::process::exit(fluxer_dev::dev::run_dev(&args.tasks, args.cloudflare_tunnel).await?)
|
||||
}
|
||||
Command::RustServices(args) => {
|
||||
std::process::exit(fluxer_dev::rust_services::run_rust_services(&args.services).await?)
|
||||
}
|
||||
Command::Smoke(args) => fluxer_dev::smoke::run_smoke(args.quick, args.public).await?,
|
||||
Command::Cassandra(args) => match args.command {
|
||||
CassandraCommand::Diff { output } => {
|
||||
let diff = compute_diff(None).await?;
|
||||
let output = write_diff_file(&diff, output.as_deref())?;
|
||||
println!("Wrote Cassandra schema diff to {}", output.display());
|
||||
if !diff.errors.is_empty() {
|
||||
for error in diff.errors {
|
||||
println!("error: {error}");
|
||||
}
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
CassandraCommand::Apply => {
|
||||
apply_schema(None).await?;
|
||||
}
|
||||
CassandraCommand::Verify => verify_schema(None, None).await?,
|
||||
CassandraCommand::TargetSchema => print!("{}", render_target_schema("fluxer")),
|
||||
},
|
||||
Command::Desktop(args) => match args.command {
|
||||
DesktopCommand::Install => install_desktop()?,
|
||||
DesktopCommand::Build { skip_native } => build_desktop(skip_native)?,
|
||||
DesktopCommand::Typecheck => typecheck_desktop()?,
|
||||
DesktopCommand::SmokeBuild => smoke_build_desktop()?,
|
||||
DesktopCommand::Package { builder_args } => package_desktop(&builder_args)?,
|
||||
DesktopCommand::Run {
|
||||
app_url,
|
||||
no_build,
|
||||
extra_args,
|
||||
} => {
|
||||
let extra_args: Vec<_> = extra_args.into_iter().filter(|arg| arg != "--").collect();
|
||||
run_desktop(&app_url, &extra_args, !no_build)?;
|
||||
}
|
||||
DesktopCommand::Canary {
|
||||
app_url,
|
||||
no_build,
|
||||
extra_args,
|
||||
} => {
|
||||
let extra_args: Vec<_> = extra_args.into_iter().filter(|arg| arg != "--").collect();
|
||||
run_desktop_canary(app_url.as_deref(), &extra_args, !no_build).await?;
|
||||
}
|
||||
DesktopCommand::ExecDisclaimed { program, args } => {
|
||||
fluxer_dev::disclaim::exec_disclaimed(&program, &args)?
|
||||
}
|
||||
},
|
||||
Command::LocalK8s(args) => match args.command {
|
||||
LocalK8sCommand::CreateCluster => fluxer_dev::local_k8s::create_cluster().await?,
|
||||
LocalK8sCommand::Kubectl { args } => fluxer_dev::local_k8s::run_kubectl_cli(&args)?,
|
||||
LocalK8sCommand::Helm { args } => fluxer_dev::local_k8s::run_helm_cli(&args)?,
|
||||
LocalK8sCommand::HotpatchSmoke => fluxer_dev::local_k8s::run_hotpatch_smoke().await?,
|
||||
LocalK8sCommand::HandoffRolloutSmoke => {
|
||||
fluxer_dev::local_k8s::run_handoff_rollout_smoke().await?
|
||||
}
|
||||
},
|
||||
Command::Marketing(args) => match args.command {
|
||||
MarketingCommand::PreprocessBlogImage(args) => {
|
||||
fluxer_dev::marketing::preprocess_blog_image(args)?
|
||||
}
|
||||
MarketingCommand::PreprocessBlogVideo(args) => {
|
||||
fluxer_dev::marketing::preprocess_blog_video(args)?
|
||||
}
|
||||
},
|
||||
Command::MediaProxy(args) => match args.command {
|
||||
MediaProxyCommand::BeeGifBench => fluxer_dev::media_proxy::run_bee_gif_bench().await?,
|
||||
MediaProxyCommand::Doctor {
|
||||
repair,
|
||||
base_url,
|
||||
path,
|
||||
} => {
|
||||
fluxer_dev::media_proxy::run_dev_media_doctor(repair, &base_url, path.as_deref())
|
||||
.await?;
|
||||
}
|
||||
MediaProxyCommand::SeaweedfsIntegration { isolated_store } => {
|
||||
fluxer_dev::media_proxy::run_seaweedfs_media_proxy_integration(isolated_store)
|
||||
.await?;
|
||||
}
|
||||
MediaProxyCommand::RustStressSmoke => {
|
||||
fluxer_dev::media_proxy::run_rust_stress_smoke()?;
|
||||
}
|
||||
MediaProxyCommand::StressCompare(args) => {
|
||||
std::process::exit(fluxer_dev::media_stress::run_stress_compare(args).await?)
|
||||
}
|
||||
MediaProxyCommand::SignExternalUrl(args) => {
|
||||
println!(
|
||||
"{}",
|
||||
fluxer_dev::media_stress::sign_external_url(
|
||||
&args.secret_key,
|
||||
&args.server_url,
|
||||
&args.upstream
|
||||
)?
|
||||
);
|
||||
}
|
||||
},
|
||||
Command::Tunnel(args) => match args.command {
|
||||
TunnelCommand::Configure { public_url, token } => {
|
||||
fluxer_dev::tunnel::write_cloudflare_public_url_file(&public_url)?;
|
||||
if let Some(token) = token {
|
||||
fluxer_dev::tunnel::write_cloudflare_token_file(&token)?;
|
||||
}
|
||||
}
|
||||
TunnelCommand::PrintEnv { public_url } => {
|
||||
print!("{}", fluxer_dev::tunnel::public_url_env_text(&public_url)?);
|
||||
}
|
||||
TunnelCommand::Run { token, token_file } => std::process::exit(
|
||||
fluxer_dev::tunnel::run_cloudflare_tunnel(token, token_file).await?,
|
||||
),
|
||||
},
|
||||
Command::NativeVoiceIt(args) => fluxer_dev::native_voice_it::run(args).await?,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_default_env() -> Result<()> {
|
||||
let current: BTreeMap<String, String> = std::env::vars().collect();
|
||||
let merged = merge_default_env_with_current(
|
||||
DEV_ENV_FILE.as_path(),
|
||||
DEV_LOCAL_ENV_FILE.as_path(),
|
||||
ROOT_LOCAL_ENV_FILE.as_path(),
|
||||
current,
|
||||
)?;
|
||||
for (key, value) in merged {
|
||||
unsafe {
|
||||
std::env::set_var(key, value);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::paths::ROOT;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub const LOOPBACK_HOST: &str = "127.0.0.1";
|
||||
pub const ANY_HOST: &str = "0.0.0.0";
|
||||
|
||||
pub const DEV_PROXY_PORT: u16 = 8088;
|
||||
pub const APP_PORT: u16 = 3000;
|
||||
pub const APP_PROXY_PORT: u16 = 8773;
|
||||
pub const ADMIN_PORT: u16 = 3020;
|
||||
pub const API_PORT: u16 = 8080;
|
||||
pub const GATEWAY_PORT: u16 = 8771;
|
||||
pub const GATEWAY_WEBSOCKET_PORTS: &[u16] = &[8771, 8772, 8774];
|
||||
pub const MEDIA_PROXY_PORT: u16 = 8082;
|
||||
pub const MARKETING_PORT: u16 = 3010;
|
||||
pub const LIVEKIT_PORT: u16 = 7880;
|
||||
pub const DEVMAIL_PORT: u16 = 8025;
|
||||
|
||||
pub const LOCAL_APP_URL: &str = "http://localhost:8088";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProxyRoute {
|
||||
pub prefix: &'static str,
|
||||
pub host: &'static str,
|
||||
pub port: u16,
|
||||
pub strip_prefix: bool,
|
||||
pub alternate_ports: &'static [u16],
|
||||
}
|
||||
|
||||
pub const PROXY_ROUTES: &[ProxyRoute] = &[
|
||||
ProxyRoute {
|
||||
prefix: "/api",
|
||||
host: LOOPBACK_HOST,
|
||||
port: API_PORT,
|
||||
strip_prefix: true,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/gateway",
|
||||
host: LOOPBACK_HOST,
|
||||
port: GATEWAY_PORT,
|
||||
strip_prefix: true,
|
||||
alternate_ports: &[8772, 8774],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/media",
|
||||
host: LOOPBACK_HOST,
|
||||
port: MEDIA_PROXY_PORT,
|
||||
strip_prefix: true,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/fluxer-uploads",
|
||||
host: LOOPBACK_HOST,
|
||||
port: 8333,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/livekit",
|
||||
host: "livekit",
|
||||
port: LIVEKIT_PORT,
|
||||
strip_prefix: true,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/devmail",
|
||||
host: "mailpit",
|
||||
port: DEVMAIL_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/admin",
|
||||
host: LOOPBACK_HOST,
|
||||
port: ADMIN_PORT,
|
||||
strip_prefix: true,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/marketing/branding",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PROXY_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/marketing/flags",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PROXY_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/marketing/pwa-install",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PROXY_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/marketing/screenshots",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PROXY_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/marketing",
|
||||
host: LOOPBACK_HOST,
|
||||
port: MARKETING_PORT,
|
||||
strip_prefix: true,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/assets",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/lazy-compilation-using-",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/manifest.json",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/browserconfig.xml",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/robots.txt",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/sw.js",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/sw.js.map",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/version.json",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
ProxyRoute {
|
||||
prefix: "/",
|
||||
host: LOOPBACK_HOST,
|
||||
port: APP_PROXY_PORT,
|
||||
strip_prefix: false,
|
||||
alternate_ports: &[],
|
||||
},
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RustServiceSpec {
|
||||
pub name: &'static str,
|
||||
pub package: &'static str,
|
||||
pub path: PathBuf,
|
||||
pub port_base: u16,
|
||||
}
|
||||
|
||||
pub fn rust_services() -> Vec<RustServiceSpec> {
|
||||
vec![
|
||||
RustServiceSpec {
|
||||
name: "messages",
|
||||
package: "fluxer-messages",
|
||||
path: ROOT.join("fluxer_messages"),
|
||||
port_base: 8112,
|
||||
},
|
||||
RustServiceSpec {
|
||||
name: "snowflakes",
|
||||
package: "fluxer-snowflakes",
|
||||
path: ROOT.join("fluxer_snowflakes"),
|
||||
port_base: 8120,
|
||||
},
|
||||
RustServiceSpec {
|
||||
name: "unfurl",
|
||||
package: "fluxer-unfurl",
|
||||
path: ROOT.join("fluxer_unfurl"),
|
||||
port_base: 8122,
|
||||
},
|
||||
RustServiceSpec {
|
||||
name: "users",
|
||||
package: "fluxer-users",
|
||||
path: ROOT.join("fluxer_users"),
|
||||
port_base: 8124,
|
||||
},
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::paths::{ROOT, which};
|
||||
use crate::proc::format_command;
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::{Args, ValueEnum};
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
const DEFAULT_BLOG_OUTPUT_DIR: &str = "fluxer_marketing/static/blog";
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct PreprocessBlogImageArgs {
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
#[arg(long)]
|
||||
name: String,
|
||||
#[arg(long, default_value = DEFAULT_BLOG_OUTPUT_DIR)]
|
||||
output_dir: PathBuf,
|
||||
#[arg(long, value_enum, default_value_t = ImageFallback::Png)]
|
||||
fallback: ImageFallback,
|
||||
#[arg(long, value_delimiter = ' ', default_values_t = [640, 960, 1280, 2000])]
|
||||
widths: Vec<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
pub enum ImageFallback {
|
||||
Png,
|
||||
Jpg,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct PreprocessBlogVideoArgs {
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
#[arg(long)]
|
||||
name: String,
|
||||
#[arg(long, default_value = DEFAULT_BLOG_OUTPUT_DIR)]
|
||||
output_dir: PathBuf,
|
||||
#[arg(long, default_value_t = 640)]
|
||||
max_width: u32,
|
||||
#[arg(long)]
|
||||
fps: Option<String>,
|
||||
#[arg(long, default_value = "0.5")]
|
||||
poster_time: String,
|
||||
#[arg(long, value_enum, default_value_t = VideoAudio::Keep)]
|
||||
audio: VideoAudio,
|
||||
#[arg(long, default_value_t = 24)]
|
||||
mp4_crf: u32,
|
||||
#[arg(long, default_value_t = 34)]
|
||||
webm_crf: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
pub enum VideoAudio {
|
||||
Keep,
|
||||
None,
|
||||
}
|
||||
|
||||
pub fn preprocess_blog_image(args: PreprocessBlogImageArgs) -> Result<()> {
|
||||
require_tools(&["magick", "cwebp", "ffmpeg"])?;
|
||||
validate_asset_name(&args.name)?;
|
||||
ensure_input_file(&args.input)?;
|
||||
if args.widths.is_empty() {
|
||||
bail!("--widths must include at least one width");
|
||||
}
|
||||
let (original_width, _) = image::image_dimensions(&args.input)
|
||||
.with_context(|| format!("could not read image dimensions: {}", args.input.display()))?;
|
||||
if original_width == 0 {
|
||||
bail!("could not read image width: {}", args.input.display());
|
||||
}
|
||||
|
||||
fs::create_dir_all(&args.output_dir)?;
|
||||
for width in target_widths(original_width, &args.widths)? {
|
||||
let base = args.output_dir.join(format!("{}-{width}", args.name));
|
||||
let fallback_file = match args.fallback {
|
||||
ImageFallback::Png => {
|
||||
let output = base.with_extension("png");
|
||||
run(&image_magick_png_command(&args.input, width, &output)?)?;
|
||||
output
|
||||
}
|
||||
ImageFallback::Jpg => {
|
||||
let output = base.with_extension("jpg");
|
||||
run(&image_magick_jpg_command(&args.input, width, &output)?)?;
|
||||
output
|
||||
}
|
||||
};
|
||||
run(&cwebp_command(
|
||||
&fallback_file,
|
||||
&base.with_extension("webp"),
|
||||
)?)?;
|
||||
run(&avif_still_command(
|
||||
&fallback_file,
|
||||
&base.with_extension("avif"),
|
||||
)?)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn preprocess_blog_video(args: PreprocessBlogVideoArgs) -> Result<()> {
|
||||
require_tools(&["ffmpeg", "ffprobe"])?;
|
||||
validate_asset_name(&args.name)?;
|
||||
ensure_input_file(&args.input)?;
|
||||
if args.max_width < 2 {
|
||||
bail!("--max-width must be an integer greater than 1");
|
||||
}
|
||||
ensure_video_stream(&args.input)?;
|
||||
|
||||
fs::create_dir_all(&args.output_dir)?;
|
||||
let base = args.output_dir.join(&args.name);
|
||||
let profile = VideoProfile::from_args(&args);
|
||||
run(&mp4_command(
|
||||
&args.input,
|
||||
&base.with_extension("mp4"),
|
||||
&profile,
|
||||
)?)?;
|
||||
run(&webm_command(
|
||||
&args.input,
|
||||
&base.with_extension("webm"),
|
||||
&profile,
|
||||
)?)?;
|
||||
run(&poster_command(
|
||||
&args.input,
|
||||
&base.with_file_name(format!("{}-poster.jpg", args.name)),
|
||||
&profile,
|
||||
)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_tools(tools: &[&str]) -> Result<()> {
|
||||
for tool in tools {
|
||||
if which(tool).is_none() {
|
||||
bail!("missing required tool: {tool}");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_asset_name(name: &str) -> Result<()> {
|
||||
if name.trim().is_empty() {
|
||||
bail!("--name must not be empty");
|
||||
}
|
||||
if name.contains('/') || name.contains('\\') {
|
||||
bail!("--name must be a file name, not a path");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_input_file(input: &Path) -> Result<()> {
|
||||
if !input.is_file() {
|
||||
bail!("input file not found: {}", input.display());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_video_stream(input: &Path) -> Result<()> {
|
||||
let status = Command::new("ffprobe")
|
||||
.args([
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-show_entries",
|
||||
"stream=width,height",
|
||||
"-of",
|
||||
"csv=p=0",
|
||||
])
|
||||
.arg(input)
|
||||
.status()
|
||||
.with_context(|| format!("failed to run ffprobe for {}", input.display()))?;
|
||||
if !status.success() {
|
||||
bail!("could not read video stream: {}", input.display());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn target_widths(original_width: u32, requested_widths: &[u32]) -> Result<Vec<u32>> {
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut widths = Vec::new();
|
||||
for requested in requested_widths {
|
||||
if *requested == 0 {
|
||||
bail!("invalid width: {requested}");
|
||||
}
|
||||
let width = (*requested).min(original_width);
|
||||
if seen.insert(width) {
|
||||
widths.push(width);
|
||||
}
|
||||
}
|
||||
Ok(widths)
|
||||
}
|
||||
|
||||
fn image_magick_png_command(input: &Path, width: u32, output: &Path) -> Result<Vec<String>> {
|
||||
Ok(vec![
|
||||
"magick".to_owned(),
|
||||
path_arg(input),
|
||||
"-auto-orient".to_owned(),
|
||||
"-resize".to_owned(),
|
||||
format!("{width}x>"),
|
||||
"-strip".to_owned(),
|
||||
"-alpha".to_owned(),
|
||||
"off".to_owned(),
|
||||
format!("PNG24:{}", path_arg(output)),
|
||||
])
|
||||
}
|
||||
|
||||
fn image_magick_jpg_command(input: &Path, width: u32, output: &Path) -> Result<Vec<String>> {
|
||||
Ok(vec![
|
||||
"magick".to_owned(),
|
||||
path_arg(input),
|
||||
"-auto-orient".to_owned(),
|
||||
"-resize".to_owned(),
|
||||
format!("{width}x>"),
|
||||
"-strip".to_owned(),
|
||||
"-alpha".to_owned(),
|
||||
"remove".to_owned(),
|
||||
"-background".to_owned(),
|
||||
"white".to_owned(),
|
||||
"-quality".to_owned(),
|
||||
"84".to_owned(),
|
||||
"-interlace".to_owned(),
|
||||
"Plane".to_owned(),
|
||||
path_arg(output),
|
||||
])
|
||||
}
|
||||
|
||||
fn cwebp_command(input: &Path, output: &Path) -> Result<Vec<String>> {
|
||||
Ok(vec![
|
||||
"cwebp".to_owned(),
|
||||
"-quiet".to_owned(),
|
||||
"-q".to_owned(),
|
||||
"82".to_owned(),
|
||||
"-m".to_owned(),
|
||||
"6".to_owned(),
|
||||
"-af".to_owned(),
|
||||
path_arg(input),
|
||||
"-o".to_owned(),
|
||||
path_arg(output),
|
||||
])
|
||||
}
|
||||
|
||||
fn avif_still_command(input: &Path, output: &Path) -> Result<Vec<String>> {
|
||||
Ok(vec![
|
||||
"ffmpeg".to_owned(),
|
||||
"-hide_banner".to_owned(),
|
||||
"-loglevel".to_owned(),
|
||||
"error".to_owned(),
|
||||
"-y".to_owned(),
|
||||
"-i".to_owned(),
|
||||
path_arg(input),
|
||||
"-frames:v".to_owned(),
|
||||
"1".to_owned(),
|
||||
"-c:v".to_owned(),
|
||||
"libaom-av1".to_owned(),
|
||||
"-still-picture".to_owned(),
|
||||
"1".to_owned(),
|
||||
"-crf".to_owned(),
|
||||
"34".to_owned(),
|
||||
"-cpu-used".to_owned(),
|
||||
"6".to_owned(),
|
||||
path_arg(output),
|
||||
])
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VideoProfile {
|
||||
scale_filter: String,
|
||||
video_filter: String,
|
||||
audio_map: Vec<String>,
|
||||
mp4_audio: Vec<String>,
|
||||
webm_audio: Vec<String>,
|
||||
mp4_crf: String,
|
||||
webm_crf: String,
|
||||
poster_time: String,
|
||||
}
|
||||
|
||||
impl VideoProfile {
|
||||
fn from_args(args: &PreprocessBlogVideoArgs) -> Self {
|
||||
let scale_filter = format!("scale='trunc(min({},iw)/2)*2':-2", args.max_width);
|
||||
let video_filter = match &args.fps {
|
||||
Some(fps) if !fps.is_empty() => format!("{scale_filter},fps={fps},format=yuv420p"),
|
||||
_ => format!("{scale_filter},format=yuv420p"),
|
||||
};
|
||||
let (audio_map, mp4_audio, webm_audio) = match args.audio {
|
||||
VideoAudio::Keep => (
|
||||
strings(&["-map", "0:v:0", "-map", "0:a?"]),
|
||||
strings(&["-c:a", "aac", "-b:a", "128k"]),
|
||||
strings(&["-c:a", "libopus", "-b:a", "96k"]),
|
||||
),
|
||||
VideoAudio::None => (
|
||||
strings(&["-map", "0:v:0"]),
|
||||
strings(&["-an"]),
|
||||
strings(&["-an"]),
|
||||
),
|
||||
};
|
||||
Self {
|
||||
scale_filter,
|
||||
video_filter,
|
||||
audio_map,
|
||||
mp4_audio,
|
||||
webm_audio,
|
||||
mp4_crf: args.mp4_crf.to_string(),
|
||||
webm_crf: args.webm_crf.to_string(),
|
||||
poster_time: args.poster_time.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn mp4_command(input: &Path, output: &Path, profile: &VideoProfile) -> Result<Vec<String>> {
|
||||
let mut command = strings(&["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i"]);
|
||||
command.push(path_arg(input));
|
||||
command.extend(profile.audio_map.iter().cloned());
|
||||
command.extend(strings(&[
|
||||
"-vf",
|
||||
&profile.video_filter,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"slow",
|
||||
"-crf",
|
||||
&profile.mp4_crf,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
]));
|
||||
command.extend(profile.mp4_audio.iter().cloned());
|
||||
command.push(path_arg(output));
|
||||
Ok(command)
|
||||
}
|
||||
|
||||
fn webm_command(input: &Path, output: &Path, profile: &VideoProfile) -> Result<Vec<String>> {
|
||||
let mut command = strings(&["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i"]);
|
||||
command.push(path_arg(input));
|
||||
command.extend(profile.audio_map.iter().cloned());
|
||||
command.extend(strings(&[
|
||||
"-vf",
|
||||
&profile.video_filter,
|
||||
"-c:v",
|
||||
"libvpx-vp9",
|
||||
"-b:v",
|
||||
"0",
|
||||
"-crf",
|
||||
&profile.webm_crf,
|
||||
"-deadline",
|
||||
"good",
|
||||
"-cpu-used",
|
||||
"4",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
]));
|
||||
command.extend(profile.webm_audio.iter().cloned());
|
||||
command.push(path_arg(output));
|
||||
Ok(command)
|
||||
}
|
||||
|
||||
fn poster_command(input: &Path, output: &Path, profile: &VideoProfile) -> Result<Vec<String>> {
|
||||
Ok(vec![
|
||||
"ffmpeg".to_owned(),
|
||||
"-hide_banner".to_owned(),
|
||||
"-loglevel".to_owned(),
|
||||
"error".to_owned(),
|
||||
"-y".to_owned(),
|
||||
"-ss".to_owned(),
|
||||
profile.poster_time.clone(),
|
||||
"-i".to_owned(),
|
||||
path_arg(input),
|
||||
"-map".to_owned(),
|
||||
"0:v:0".to_owned(),
|
||||
"-frames:v".to_owned(),
|
||||
"1".to_owned(),
|
||||
"-vf".to_owned(),
|
||||
profile.scale_filter.clone(),
|
||||
"-q:v".to_owned(),
|
||||
"3".to_owned(),
|
||||
path_arg(output),
|
||||
])
|
||||
}
|
||||
|
||||
fn run(args: &[String]) -> Result<()> {
|
||||
println!("$ {}", format_command(args));
|
||||
let status = Command::new(&args[0])
|
||||
.args(&args[1..])
|
||||
.current_dir(ROOT.as_path())
|
||||
.status()
|
||||
.with_context(|| format!("failed to run {}", format_command(args)))?;
|
||||
if !status.success() {
|
||||
bail!(
|
||||
"command failed with exit code {}: {}",
|
||||
status.code().unwrap_or(1),
|
||||
format_command(args)
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn strings(values: &[&str]) -> Vec<String> {
|
||||
values.iter().map(|value| (*value).to_owned()).collect()
|
||||
}
|
||||
|
||||
fn path_arg(path: &Path) -> String {
|
||||
path.display().to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn target_widths_clamp_and_deduplicate_in_request_order() {
|
||||
assert_eq!(
|
||||
target_widths(1000, &[640, 960, 1280, 1000, 960]).unwrap(),
|
||||
vec![640, 960, 1000]
|
||||
);
|
||||
assert!(target_widths(1000, &[0]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_png_fallback_command() {
|
||||
let command =
|
||||
image_magick_png_command(Path::new("in.png"), 640, Path::new("out.png")).unwrap();
|
||||
assert_eq!(command[0], "magick");
|
||||
assert!(command.contains(&"640x>".to_owned()));
|
||||
assert_eq!(command.last().unwrap(), "PNG24:out.png");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_jpg_fallback_command() {
|
||||
let command =
|
||||
image_magick_jpg_command(Path::new("in.png"), 640, Path::new("out.jpg")).unwrap();
|
||||
assert!(command.contains(&"-background".to_owned()));
|
||||
assert!(command.contains(&"white".to_owned()));
|
||||
assert_eq!(command.last().unwrap(), "out.jpg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_video_profile_with_fps_and_no_audio() {
|
||||
let profile = VideoProfile::from_args(&PreprocessBlogVideoArgs {
|
||||
input: PathBuf::from("in.gif"),
|
||||
name: "clip".to_owned(),
|
||||
output_dir: PathBuf::from("out"),
|
||||
max_width: 640,
|
||||
fps: Some("15".to_owned()),
|
||||
poster_time: "0.5".to_owned(),
|
||||
audio: VideoAudio::None,
|
||||
mp4_crf: 24,
|
||||
webm_crf: 34,
|
||||
});
|
||||
assert_eq!(
|
||||
profile.video_filter,
|
||||
"scale='trunc(min(640,iw)/2)*2':-2,fps=15,format=yuv420p"
|
||||
);
|
||||
assert_eq!(profile.audio_map, strings(&["-map", "0:v:0"]));
|
||||
assert_eq!(profile.mp4_audio, strings(&["-an"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_video_encode_commands() {
|
||||
let profile = VideoProfile::from_args(&PreprocessBlogVideoArgs {
|
||||
input: PathBuf::from("in.gif"),
|
||||
name: "clip".to_owned(),
|
||||
output_dir: PathBuf::from("out"),
|
||||
max_width: 320,
|
||||
fps: None,
|
||||
poster_time: "1.25".to_owned(),
|
||||
audio: VideoAudio::Keep,
|
||||
mp4_crf: 20,
|
||||
webm_crf: 30,
|
||||
});
|
||||
let mp4 = mp4_command(Path::new("in.gif"), Path::new("clip.mp4"), &profile).unwrap();
|
||||
assert!(mp4.contains(&"libx264".to_owned()));
|
||||
assert!(mp4.contains(&"+faststart".to_owned()));
|
||||
assert_eq!(mp4.last().unwrap(), "clip.mp4");
|
||||
|
||||
let poster =
|
||||
poster_command(Path::new("in.gif"), Path::new("clip-poster.jpg"), &profile).unwrap();
|
||||
assert!(poster.contains(&"1.25".to_owned()));
|
||||
assert_eq!(poster.last().unwrap(), "clip-poster.jpg");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,801 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use clap::Args;
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use image::{AnimationDecoder, RgbaImage, codecs::gif::GifDecoder, imageops::FilterType};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Sha256;
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::{BufReader, Cursor};
|
||||
use std::process::Command;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
const DEFAULT_PREFIXES: &[&str] = &[
|
||||
"attachments/",
|
||||
"emojis/",
|
||||
"stickers/",
|
||||
"avatars/",
|
||||
"icons/",
|
||||
"banners/",
|
||||
];
|
||||
const KNOWN_EXTS: &[&str] = &[
|
||||
"png", "jpg", "jpeg", "webp", "gif", "apng", "avif", "heic", "heif", "jxl",
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct StressCompareArgs {
|
||||
#[arg(long, help = "v1 base URL, for example http://media-proxy:8080")]
|
||||
pub v1: String,
|
||||
#[arg(long, help = "v2 base URL, for example http://10.244.147.209:8080")]
|
||||
pub v2: String,
|
||||
#[arg(long, default_value = "fluxer")]
|
||||
pub bucket: String,
|
||||
#[arg(
|
||||
long,
|
||||
env = "FLUXER_S3_ENDPOINT",
|
||||
default_value = "https://ewr1.vultrobjects.com"
|
||||
)]
|
||||
pub endpoint: String,
|
||||
#[arg(long, env = "FLUXER_S3_REGION", default_value = "ewr1")]
|
||||
pub region: String,
|
||||
#[arg(long, env = "FLUXER_S3_ACCESS_KEY_ID")]
|
||||
pub access_key: Option<String>,
|
||||
#[arg(long, env = "FLUXER_S3_SECRET_ACCESS_KEY")]
|
||||
pub secret_key: Option<String>,
|
||||
#[arg(long, action = clap::ArgAction::Append)]
|
||||
pub prefix: Vec<String>,
|
||||
#[arg(long, default_value_t = 20)]
|
||||
pub per_prefix: usize,
|
||||
#[arg(long, default_value = "plain,resize,format,resize+format")]
|
||||
pub matrix: String,
|
||||
#[arg(long, default_value_t = 256)]
|
||||
pub size: u32,
|
||||
#[arg(long, default_value = "webp")]
|
||||
pub format: String,
|
||||
#[arg(long, default_value_t = 8)]
|
||||
pub concurrency: usize,
|
||||
#[arg(long, default_value_t = 30.0)]
|
||||
pub timeout: f64,
|
||||
#[arg(long, default_value_t = 42)]
|
||||
pub seed: u64,
|
||||
#[arg(long, default_value = "-", help = "JSON report path, or - for stdout")]
|
||||
pub report: String,
|
||||
#[arg(long, default_value_t = 40)]
|
||||
pub max_issues_print: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct SignExternalUrlArgs {
|
||||
#[arg(long)]
|
||||
pub secret_key: String,
|
||||
#[arg(long)]
|
||||
pub server_url: String,
|
||||
pub upstream: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||
pub struct StressCase {
|
||||
pub label: String,
|
||||
pub url_path: String,
|
||||
#[serde(skip_serializing_if = "String::is_empty")]
|
||||
pub query: String,
|
||||
pub expect_image: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
pub struct StressResult {
|
||||
pub case: StressCase,
|
||||
pub v1_status: i32,
|
||||
pub v2_status: i32,
|
||||
pub v1_ct: String,
|
||||
pub v2_ct: String,
|
||||
pub v1_size: usize,
|
||||
pub v2_size: usize,
|
||||
pub issues: Vec<String>,
|
||||
pub elapsed_v1_ms: f64,
|
||||
pub elapsed_v2_ms: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
|
||||
pub struct PrefixSummary {
|
||||
pub ok: u64,
|
||||
pub fail: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, PartialEq)]
|
||||
pub struct StressSummary {
|
||||
pub total: usize,
|
||||
pub issues: usize,
|
||||
pub by_prefix: BTreeMap<String, PrefixSummary>,
|
||||
pub issues_sample: Vec<StressResult>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FetchResult {
|
||||
status: i32,
|
||||
content_type: String,
|
||||
body: Vec<u8>,
|
||||
elapsed_ms: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct AwsListObjects {
|
||||
#[serde(rename = "Contents", default)]
|
||||
contents: Vec<AwsObject>,
|
||||
#[serde(rename = "IsTruncated", default)]
|
||||
is_truncated: bool,
|
||||
#[serde(rename = "NextContinuationToken")]
|
||||
next_continuation_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct AwsObject {
|
||||
#[serde(rename = "Key")]
|
||||
key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct StableRng(u64);
|
||||
|
||||
impl StableRng {
|
||||
fn new(seed: u64) -> Self {
|
||||
Self(seed)
|
||||
}
|
||||
|
||||
fn next_below(&mut self, upper: usize) -> usize {
|
||||
self.0 = self
|
||||
.0
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
((self.0 >> 32) as usize) % upper
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_stress_compare(args: StressCompareArgs) -> Result<i32> {
|
||||
let access_key = args
|
||||
.access_key
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
.context("missing S3 creds: pass --access-key or set FLUXER_S3_ACCESS_KEY_ID")?;
|
||||
let secret_key = args
|
||||
.secret_key
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
.context("missing S3 creds: pass --secret-key or set FLUXER_S3_SECRET_ACCESS_KEY")?;
|
||||
|
||||
let prefixes = if args.prefix.is_empty() {
|
||||
DEFAULT_PREFIXES
|
||||
.iter()
|
||||
.map(|value| (*value).to_owned())
|
||||
.collect()
|
||||
} else {
|
||||
args.prefix.clone()
|
||||
};
|
||||
let mut rng = StableRng::new(args.seed);
|
||||
let mut cases = Vec::new();
|
||||
for prefix in &prefixes {
|
||||
let keys = list_random_keys(
|
||||
&RandomKeyListRequest {
|
||||
endpoint: &args.endpoint,
|
||||
access_key,
|
||||
secret_key,
|
||||
region: &args.region,
|
||||
bucket: &args.bucket,
|
||||
prefix,
|
||||
want: args.per_prefix,
|
||||
},
|
||||
&mut rng,
|
||||
)?;
|
||||
for key in keys {
|
||||
cases.extend(cases_for_key(&key, &args.matrix, args.size, &args.format));
|
||||
}
|
||||
}
|
||||
eprintln!("# {} cases across {} prefixes", cases.len(), prefixes.len());
|
||||
|
||||
let results = compare_cases(
|
||||
cases,
|
||||
&args.v1,
|
||||
&args.v2,
|
||||
args.timeout,
|
||||
args.concurrency.max(1),
|
||||
)
|
||||
.await?;
|
||||
let summary = summarize_results(&results, args.max_issues_print);
|
||||
let output = serde_json::to_string_pretty(&summary)?;
|
||||
if args.report == "-" {
|
||||
println!("{output}");
|
||||
} else {
|
||||
std::fs::write(&args.report, output)
|
||||
.with_context(|| format!("failed to write {}", args.report))?;
|
||||
}
|
||||
Ok(summary.issues.min(255) as i32)
|
||||
}
|
||||
|
||||
pub fn sign_external_url(secret_key: &str, server_url: &str, upstream: &str) -> Result<String> {
|
||||
let path = format!("v2/{}", URL_SAFE_NO_PAD.encode(upstream.as_bytes()));
|
||||
let mut mac = Hmac::<Sha256>::new_from_slice(secret_key.as_bytes())
|
||||
.context("failed to create HMAC signer")?;
|
||||
mac.update(path.as_bytes());
|
||||
let signature = URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes());
|
||||
Ok(format!(
|
||||
"{}/external/{}/{}",
|
||||
server_url.trim_end_matches('/'),
|
||||
signature,
|
||||
path
|
||||
))
|
||||
}
|
||||
|
||||
struct RandomKeyListRequest<'a> {
|
||||
endpoint: &'a str,
|
||||
access_key: &'a str,
|
||||
secret_key: &'a str,
|
||||
region: &'a str,
|
||||
bucket: &'a str,
|
||||
prefix: &'a str,
|
||||
want: usize,
|
||||
}
|
||||
|
||||
fn list_random_keys(
|
||||
request: &RandomKeyListRequest<'_>,
|
||||
rng: &mut StableRng,
|
||||
) -> Result<Vec<String>> {
|
||||
if request.want == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut keys = Vec::new();
|
||||
let mut seen = 0usize;
|
||||
let mut continuation_token = None;
|
||||
let scan_limit = request.want.saturating_mul(50).max(5000);
|
||||
|
||||
loop {
|
||||
let page = list_objects_page(
|
||||
request.endpoint,
|
||||
request.access_key,
|
||||
request.secret_key,
|
||||
request.region,
|
||||
request.bucket,
|
||||
request.prefix,
|
||||
continuation_token.as_deref(),
|
||||
)?;
|
||||
for object in page.contents {
|
||||
seen += 1;
|
||||
if keys.len() < request.want {
|
||||
keys.push(object.key);
|
||||
} else {
|
||||
let index = rng.next_below(seen);
|
||||
if index < request.want {
|
||||
keys[index] = object.key;
|
||||
}
|
||||
}
|
||||
}
|
||||
if seen >= scan_limit || !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
continuation_token = page.next_continuation_token;
|
||||
if continuation_token.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
fn list_objects_page(
|
||||
endpoint: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
region: &str,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<&str>,
|
||||
) -> Result<AwsListObjects> {
|
||||
let mut command = Command::new("aws");
|
||||
command
|
||||
.arg("--no-cli-pager")
|
||||
.arg("--endpoint-url")
|
||||
.arg(endpoint)
|
||||
.arg("--region")
|
||||
.arg(region)
|
||||
.arg("s3api")
|
||||
.arg("list-objects-v2")
|
||||
.arg("--bucket")
|
||||
.arg(bucket)
|
||||
.arg("--prefix")
|
||||
.arg(prefix)
|
||||
.arg("--max-keys")
|
||||
.arg("1000")
|
||||
.arg("--output")
|
||||
.arg("json")
|
||||
.env("AWS_ACCESS_KEY_ID", access_key)
|
||||
.env("AWS_SECRET_ACCESS_KEY", secret_key)
|
||||
.env("AWS_DEFAULT_REGION", region)
|
||||
.env("AWS_EC2_METADATA_DISABLED", "true");
|
||||
if let Some(token) = continuation_token {
|
||||
command.arg("--continuation-token").arg(token);
|
||||
}
|
||||
let output = command.output().context("failed to run aws s3api")?;
|
||||
if !output.status.success() {
|
||||
bail!(
|
||||
"aws s3api list-objects-v2 failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
);
|
||||
}
|
||||
serde_json::from_slice(&output.stdout).context("failed to parse aws s3api JSON")
|
||||
}
|
||||
|
||||
pub fn cases_for_key(key: &str, matrix: &str, size: u32, format: &str) -> Vec<StressCase> {
|
||||
let Some(url_path) = url_for_key(key) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let is_attachment = key.starts_with("attachments/");
|
||||
let mut cases = Vec::new();
|
||||
for variant in matrix.split(',').map(str::trim).filter(|v| !v.is_empty()) {
|
||||
match variant {
|
||||
"plain" => cases.push(StressCase {
|
||||
label: format!("{key}|plain"),
|
||||
url_path: url_path.clone(),
|
||||
query: String::new(),
|
||||
expect_image: true,
|
||||
}),
|
||||
"resize" => cases.push(StressCase {
|
||||
label: format!("{key}|resize{size}"),
|
||||
url_path: url_path.clone(),
|
||||
query: resize_query(is_attachment, size),
|
||||
expect_image: true,
|
||||
}),
|
||||
"format" => cases.push(StressCase {
|
||||
label: format!("{key}|fmt={format}"),
|
||||
url_path: url_path.clone(),
|
||||
query: format!("format={format}"),
|
||||
expect_image: true,
|
||||
}),
|
||||
"resize+format" if is_attachment => cases.push(StressCase {
|
||||
label: format!("{key}|w{size}+{format}"),
|
||||
url_path: url_path.clone(),
|
||||
query: format!("format={format}&width={size}&height={size}"),
|
||||
expect_image: true,
|
||||
}),
|
||||
"resize+format" => cases.push(StressCase {
|
||||
label: format!("{key}|s{size}+{format}"),
|
||||
url_path: url_path.clone(),
|
||||
query: format!("size={size}&format={format}"),
|
||||
expect_image: true,
|
||||
}),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
cases
|
||||
}
|
||||
|
||||
pub fn url_for_key(key: &str) -> Option<String> {
|
||||
let parts = key.split('/').collect::<Vec<_>>();
|
||||
let prefix = parts.first().copied()?;
|
||||
match prefix {
|
||||
"emojis" | "stickers" if parts.len() == 2 => Some(format!(
|
||||
"/{prefix}/{}",
|
||||
quote_component(&ensure_ext(parts[1], "webp"))
|
||||
)),
|
||||
"attachments" if parts.len() == 4 => Some(format!(
|
||||
"/attachments/{}/{}/{}",
|
||||
quote_component(parts[1]),
|
||||
quote_component(parts[2]),
|
||||
quote_component(parts[3])
|
||||
)),
|
||||
"avatars" | "icons" | "banners" | "splashes" | "embed-splashes" if parts.len() == 3 => {
|
||||
Some(format!(
|
||||
"/{prefix}/{}/{}",
|
||||
quote_component(parts[1]),
|
||||
quote_component(&ensure_ext(parts[2], "webp"))
|
||||
))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_ext(name: &str, default: &str) -> String {
|
||||
let lower = name.to_ascii_lowercase();
|
||||
if KNOWN_EXTS
|
||||
.iter()
|
||||
.any(|extension| lower.ends_with(&format!(".{extension}")))
|
||||
{
|
||||
name.to_owned()
|
||||
} else {
|
||||
format!("{name}.{default}")
|
||||
}
|
||||
}
|
||||
|
||||
fn quote_component(value: &str) -> String {
|
||||
urlencoding::encode(value).into_owned()
|
||||
}
|
||||
|
||||
fn resize_query(is_attachment: bool, size: u32) -> String {
|
||||
if is_attachment {
|
||||
format!("width={size}&height={size}")
|
||||
} else {
|
||||
format!("size={size}")
|
||||
}
|
||||
}
|
||||
|
||||
async fn compare_cases(
|
||||
cases: Vec<StressCase>,
|
||||
v1_base: &str,
|
||||
v2_base: &str,
|
||||
timeout: f64,
|
||||
concurrency: usize,
|
||||
) -> Result<Vec<StressResult>> {
|
||||
let client = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(Duration::from_secs_f64(timeout.max(0.1)))
|
||||
.build()?;
|
||||
let mut pending = cases.into_iter();
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut results = Vec::new();
|
||||
let mut submitted = 0usize;
|
||||
|
||||
loop {
|
||||
while join_set.len() < concurrency {
|
||||
let Some(case) = pending.next() else {
|
||||
break;
|
||||
};
|
||||
let client = client.clone();
|
||||
let v1_base = v1_base.to_owned();
|
||||
let v2_base = v2_base.to_owned();
|
||||
join_set.spawn(async move { compare_one(client, case, v1_base, v2_base).await });
|
||||
submitted += 1;
|
||||
}
|
||||
let Some(joined) = join_set.join_next().await else {
|
||||
break;
|
||||
};
|
||||
let result = joined.context("stress compare worker panicked")?;
|
||||
results.push(result);
|
||||
if results.len() % 25 == 0 {
|
||||
let issue_count = results
|
||||
.iter()
|
||||
.filter(|result| !result.issues.is_empty())
|
||||
.count();
|
||||
eprintln!(
|
||||
"# {}/{} done, {} issues so far",
|
||||
results.len(),
|
||||
submitted + pending.len(),
|
||||
issue_count
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn compare_one(
|
||||
client: reqwest::Client,
|
||||
case: StressCase,
|
||||
v1_base: String,
|
||||
v2_base: String,
|
||||
) -> StressResult {
|
||||
let v1 = fetch(&client, &v1_base, &case.url_path, &case.query).await;
|
||||
let v2 = fetch(&client, &v2_base, &case.url_path, &case.query).await;
|
||||
let mut result = StressResult {
|
||||
case,
|
||||
v1_status: v1.status,
|
||||
v2_status: v2.status,
|
||||
v1_ct: v1.content_type,
|
||||
v2_ct: v2.content_type,
|
||||
v1_size: v1.body.len(),
|
||||
v2_size: v2.body.len(),
|
||||
issues: Vec::new(),
|
||||
elapsed_v1_ms: v1.elapsed_ms,
|
||||
elapsed_v2_ms: v2.elapsed_ms,
|
||||
};
|
||||
|
||||
if result.v1_status != result.v2_status {
|
||||
result.issues.push(format!(
|
||||
"status mismatch v1={} v2={}",
|
||||
result.v1_status, result.v2_status
|
||||
));
|
||||
return result;
|
||||
}
|
||||
if result.v1_status >= 400 {
|
||||
return result;
|
||||
}
|
||||
if !result.case.expect_image {
|
||||
if v1.body != v2.body {
|
||||
result.issues.push(format!(
|
||||
"body diverges v1={}B v2={}B",
|
||||
result.v1_size, result.v2_size
|
||||
));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
let (a_image, a_frames, a_error) = decode_image(&v1.body);
|
||||
let (b_image, b_frames, b_error) = decode_image(&v2.body);
|
||||
match (a_error.as_deref(), b_error.as_deref()) {
|
||||
(Some(a_error), None) => result
|
||||
.issues
|
||||
.push(format!("v1 decode failed but v2 ok: {a_error}")),
|
||||
(None, Some(b_error)) => result.issues.push(format!(
|
||||
"v2 decode failed: {b_error} (v1 ok, {a_frames} frame(s))"
|
||||
)),
|
||||
(Some(_), Some(_)) => {
|
||||
if result.v1_size.abs_diff(result.v2_size) > result.v1_size.saturating_div(4).max(256) {
|
||||
result.issues.push(format!(
|
||||
"both decode-failed, size differs ({} vs {})",
|
||||
result.v1_size, result.v2_size
|
||||
));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
(None, None) => {}
|
||||
}
|
||||
|
||||
if let (Some(a_image), Some(b_image)) = (a_image, b_image) {
|
||||
if a_frames != b_frames {
|
||||
result
|
||||
.issues
|
||||
.push(format!("frame count v1={a_frames} v2={b_frames}"));
|
||||
}
|
||||
let similarity = pixel_similarity(&a_image, &b_image);
|
||||
if similarity > 0.08 {
|
||||
result.issues.push(format!(
|
||||
"pixel diff {similarity:.3} (v1 size {}x{} v2 size {}x{})",
|
||||
a_image.width(),
|
||||
a_image.height(),
|
||||
b_image.width(),
|
||||
b_image.height()
|
||||
));
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn fetch(client: &reqwest::Client, base: &str, path: &str, query: &str) -> FetchResult {
|
||||
let url = format!(
|
||||
"{}{}{}",
|
||||
base.trim_end_matches('/'),
|
||||
path,
|
||||
if query.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("?{query}")
|
||||
}
|
||||
);
|
||||
let started = Instant::now();
|
||||
match client.get(url).send().await {
|
||||
Ok(response) => {
|
||||
let status = response.status().as_u16() as i32;
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
match response.bytes().await {
|
||||
Ok(body) => FetchResult {
|
||||
status,
|
||||
content_type,
|
||||
body: body.to_vec(),
|
||||
elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
|
||||
},
|
||||
Err(error) => FetchResult {
|
||||
status: -1,
|
||||
content_type: format!("<error: {}: {error}>", error_kind(&error)),
|
||||
body: Vec::new(),
|
||||
elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
|
||||
},
|
||||
}
|
||||
}
|
||||
Err(error) => FetchResult {
|
||||
status: -1,
|
||||
content_type: format!("<error: {}: {error}>", error_kind(&error)),
|
||||
body: Vec::new(),
|
||||
elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"Timeout"
|
||||
} else if error.is_connect() {
|
||||
"Connect"
|
||||
} else if error.is_decode() {
|
||||
"Decode"
|
||||
} else {
|
||||
"Request"
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_image(data: &[u8]) -> (Option<RgbaImage>, usize, Option<String>) {
|
||||
if data.starts_with(b"GIF87a") || data.starts_with(b"GIF89a") {
|
||||
return decode_gif(data);
|
||||
}
|
||||
match image::load_from_memory(data) {
|
||||
Ok(image) => (Some(image.to_rgba8()), 1, None),
|
||||
Err(error) => (None, 0, Some(error.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_gif(data: &[u8]) -> (Option<RgbaImage>, usize, Option<String>) {
|
||||
let reader = BufReader::new(Cursor::new(data));
|
||||
let decoder = match GifDecoder::new(reader) {
|
||||
Ok(decoder) => decoder,
|
||||
Err(error) => return (None, 0, Some(error.to_string())),
|
||||
};
|
||||
match decoder.into_frames().collect_frames() {
|
||||
Ok(frames) => {
|
||||
let first = frames.first().map(|frame| frame.buffer().clone());
|
||||
(first, frames.len(), None)
|
||||
}
|
||||
Err(error) => (None, 0, Some(error.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
fn pixel_similarity(a: &RgbaImage, b: &RgbaImage) -> f64 {
|
||||
let b_resized;
|
||||
let b = if a.dimensions() == b.dimensions() {
|
||||
b
|
||||
} else {
|
||||
let width_diff = a.width().abs_diff(b.width());
|
||||
let height_diff = a.height().abs_diff(b.height());
|
||||
if width_diff > 8 || height_diff > 8 {
|
||||
return 1.0;
|
||||
}
|
||||
b_resized = image::imageops::resize(b, a.width(), a.height(), FilterType::Lanczos3);
|
||||
&b_resized
|
||||
};
|
||||
let sum = a
|
||||
.as_raw()
|
||||
.iter()
|
||||
.zip(b.as_raw())
|
||||
.map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs() as u64)
|
||||
.sum::<u64>();
|
||||
sum as f64 / (a.as_raw().len() as f64 * 255.0)
|
||||
}
|
||||
|
||||
pub fn summarize_results(results: &[StressResult], max_issues_print: usize) -> StressSummary {
|
||||
let mut by_prefix = BTreeMap::new();
|
||||
for result in results {
|
||||
let prefix = result
|
||||
.case
|
||||
.url_path
|
||||
.split('/')
|
||||
.nth(1)
|
||||
.unwrap_or("")
|
||||
.to_owned();
|
||||
let entry = by_prefix
|
||||
.entry(prefix)
|
||||
.or_insert(PrefixSummary { ok: 0, fail: 0 });
|
||||
if result.issues.is_empty() {
|
||||
entry.ok += 1;
|
||||
} else {
|
||||
entry.fail += 1;
|
||||
}
|
||||
}
|
||||
let issues_sample = results
|
||||
.iter()
|
||||
.filter(|result| !result.issues.is_empty())
|
||||
.take(max_issues_print)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
StressSummary {
|
||||
total: results.len(),
|
||||
issues: results
|
||||
.iter()
|
||||
.filter(|result| !result.issues.is_empty())
|
||||
.count(),
|
||||
by_prefix,
|
||||
issues_sample,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
|
||||
#[test]
|
||||
fn maps_s3_keys_to_public_routes() {
|
||||
assert_eq!(
|
||||
url_for_key("avatars/42/hash"),
|
||||
Some("/avatars/42/hash.webp".to_owned())
|
||||
);
|
||||
assert_eq!(
|
||||
url_for_key("avatars/42/hash.avif"),
|
||||
Some("/avatars/42/hash.avif".to_owned())
|
||||
);
|
||||
assert_eq!(
|
||||
url_for_key("emojis/123"),
|
||||
Some("/emojis/123.webp".to_owned())
|
||||
);
|
||||
assert_eq!(
|
||||
url_for_key("attachments/a b/c/name#.png"),
|
||||
Some("/attachments/a%20b/c/name%23.png".to_owned())
|
||||
);
|
||||
assert_eq!(url_for_key("unknown/1/2"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_transform_matrix_for_attachment_and_avatar() {
|
||||
let attachment = cases_for_key(
|
||||
"attachments/a/b/photo.png",
|
||||
"plain,resize,format,resize+format",
|
||||
128,
|
||||
"webp",
|
||||
);
|
||||
assert_eq!(attachment.len(), 4);
|
||||
assert_eq!(attachment[1].query, "width=128&height=128");
|
||||
assert_eq!(attachment[3].query, "format=webp&width=128&height=128");
|
||||
|
||||
let avatar = cases_for_key("avatars/42/hash", "resize,resize+format", 64, "webp");
|
||||
assert_eq!(avatar[0].query, "size=64");
|
||||
assert_eq!(avatar[1].query, "size=64&format=webp");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signs_external_urls_with_urlsafe_components() {
|
||||
let signed = sign_external_url(
|
||||
"benchmark-secret",
|
||||
"http://127.0.0.1:19110/",
|
||||
"https://example.test/a b.jpg",
|
||||
)
|
||||
.unwrap();
|
||||
let parts = signed.split('/').collect::<Vec<_>>();
|
||||
assert!(signed.starts_with("http://127.0.0.1:19110/external/"));
|
||||
assert_eq!(parts[5], "v2");
|
||||
assert_eq!(
|
||||
URL_SAFE_NO_PAD.decode(parts[6]).unwrap(),
|
||||
b"https://example.test/a b.jpg"
|
||||
);
|
||||
assert_eq!(URL_SAFE_NO_PAD.decode(parts[4]).unwrap().len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn summarizes_results_by_prefix() {
|
||||
let results = vec![
|
||||
result_for("/attachments/a/b/c.png", vec![]),
|
||||
result_for("/attachments/a/b/d.png", vec!["bad".to_owned()]),
|
||||
result_for("/avatars/1/a.webp", vec![]),
|
||||
];
|
||||
let summary = summarize_results(&results, 1);
|
||||
|
||||
assert_eq!(summary.total, 3);
|
||||
assert_eq!(summary.issues, 1);
|
||||
assert_eq!(
|
||||
summary.by_prefix.get("attachments"),
|
||||
Some(&PrefixSummary { ok: 1, fail: 1 })
|
||||
);
|
||||
assert_eq!(summary.issues_sample.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pixel_similarity_detects_equal_and_different_images() {
|
||||
let image_a = RgbaImage::from_pixel(2, 2, image::Rgba([0, 0, 0, 255]));
|
||||
let image_b = RgbaImage::from_pixel(2, 2, image::Rgba([0, 0, 0, 255]));
|
||||
let image_c = RgbaImage::from_pixel(2, 2, image::Rgba([255, 255, 255, 255]));
|
||||
|
||||
assert_eq!(pixel_similarity(&image_a, &image_b), 0.0);
|
||||
assert!(pixel_similarity(&image_a, &image_c) > 0.70);
|
||||
}
|
||||
|
||||
fn result_for(path: &str, issues: Vec<String>) -> StressResult {
|
||||
StressResult {
|
||||
case: StressCase {
|
||||
label: path.to_owned(),
|
||||
url_path: path.to_owned(),
|
||||
query: String::new(),
|
||||
expect_image: true,
|
||||
},
|
||||
v1_status: 200,
|
||||
v2_status: 200,
|
||||
v1_ct: "image/png".to_owned(),
|
||||
v2_ct: "image/png".to_owned(),
|
||||
v1_size: 10,
|
||||
v2_size: 10,
|
||||
issues,
|
||||
elapsed_v1_ms: 1.0,
|
||||
elapsed_v2_ms: 1.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,115 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static ROOT: LazyLock<PathBuf> = LazyLock::new(|| {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.and_then(Path::parent)
|
||||
.expect("tools/dev has a repository grandparent")
|
||||
.to_path_buf()
|
||||
});
|
||||
|
||||
pub static DEV_STATE_DIR: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join(".fluxer/dev"));
|
||||
pub static DEV_CASSANDRA_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("cassandra"));
|
||||
pub static DEV_GATEWAY_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("gateway"));
|
||||
pub static DEV_LOG_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("logs"));
|
||||
pub static DEV_SEAWEEDFS_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("seaweedfs"));
|
||||
pub static DEV_SEAWEEDFS_PID_FILE: LazyLock<PathBuf> =
|
||||
LazyLock::new(|| DEV_SEAWEEDFS_DIR.join("seaweedfs.pid"));
|
||||
pub static DEV_ENV_FILE: LazyLock<PathBuf> =
|
||||
LazyLock::new(|| ROOT.join("config/env/development.env"));
|
||||
pub static DEV_LOCAL_ENV_FILE: LazyLock<PathBuf> =
|
||||
LazyLock::new(|| ROOT.join("config/env/local.env"));
|
||||
pub static ROOT_LOCAL_ENV_FILE: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join(".env.local"));
|
||||
pub static TARGET_DIR: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join("target"));
|
||||
pub static DESKTOP_DIR: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join("fluxer_desktop"));
|
||||
pub static GATEWAY_CONFIG_DIR: LazyLock<PathBuf> =
|
||||
LazyLock::new(|| ROOT.join("fluxer_gateway/config"));
|
||||
|
||||
pub fn ensure_state_dirs() -> Result<()> {
|
||||
for path in [
|
||||
DEV_STATE_DIR.as_path(),
|
||||
DEV_CASSANDRA_DIR.as_path(),
|
||||
DEV_GATEWAY_DIR.as_path(),
|
||||
DEV_LOG_DIR.as_path(),
|
||||
DEV_SEAWEEDFS_DIR.as_path(),
|
||||
] {
|
||||
std::fs::create_dir_all(path)
|
||||
.with_context(|| format!("failed to create {}", path.display()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn ensure_writable_dev_paths() -> Result<()> {
|
||||
for path in [DEV_STATE_DIR.as_path(), TARGET_DIR.as_path()] {
|
||||
std::fs::create_dir_all(path)
|
||||
.with_context(|| format!("failed to create {}", path.display()))?;
|
||||
if is_writable(path) {
|
||||
continue;
|
||||
}
|
||||
let sudo = which("sudo").ok_or_else(|| {
|
||||
anyhow::anyhow!("{} is not writable and sudo is unavailable", path.display())
|
||||
})?;
|
||||
let owner = format!("{}:{}", current_uid(), current_gid());
|
||||
let status = Command::new(sudo)
|
||||
.args(["chown", "-R", &owner])
|
||||
.arg(path)
|
||||
.status()
|
||||
.with_context(|| format!("failed to chown {}", path.display()))?;
|
||||
if !status.success() {
|
||||
bail!("sudo chown failed for {}", path.display());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn which(name: &str) -> Option<PathBuf> {
|
||||
let path = std::env::var_os("PATH")?;
|
||||
for dir in std::env::split_paths(&path) {
|
||||
let candidate = dir.join(name);
|
||||
if candidate.is_file() {
|
||||
return Some(candidate);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn is_writable(path: &Path) -> bool {
|
||||
let probe = path.join(".fluxer-write-test");
|
||||
match std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.open(&probe)
|
||||
{
|
||||
Ok(_) => {
|
||||
let _ = std::fs::remove_file(probe);
|
||||
true
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn current_uid() -> u32 {
|
||||
unsafe { libc::geteuid() }
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn current_gid() -> u32 {
|
||||
unsafe { libc::getegid() }
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn current_uid() -> u32 {
|
||||
0
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn current_gid() -> u32 {
|
||||
0
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::env::merge_default_env_with_current;
|
||||
use crate::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::net::{TcpStream, ToSocketAddrs};
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Output, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
pub const PNPM_INSTALL_ENV: &[(&str, &str)] = &[
|
||||
("CI", "true"),
|
||||
("npm_config_child_concurrency", "2"),
|
||||
("npm_config_network_concurrency", "8"),
|
||||
];
|
||||
|
||||
pub fn format_command(args: &[impl AsRef<str>]) -> String {
|
||||
args.iter()
|
||||
.map(|arg| quote_posix(arg.as_ref()))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
fn quote_posix(value: &str) -> String {
|
||||
if value.is_empty() {
|
||||
return "''".to_owned();
|
||||
}
|
||||
if value
|
||||
.bytes()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || b"._+-/:=@%".contains(&ch))
|
||||
{
|
||||
return value.to_owned();
|
||||
}
|
||||
format!("'{}'", value.replace('\'', "'\"'\"'"))
|
||||
}
|
||||
|
||||
pub fn merged_env(
|
||||
extra: Option<&[(String, Option<String>)]>,
|
||||
load_default_env: bool,
|
||||
) -> Result<BTreeMap<String, String>> {
|
||||
let mut current: BTreeMap<String, String> = std::env::vars().collect();
|
||||
if load_default_env {
|
||||
current = merge_default_env_with_current(
|
||||
DEV_ENV_FILE.as_path(),
|
||||
DEV_LOCAL_ENV_FILE.as_path(),
|
||||
ROOT_LOCAL_ENV_FILE.as_path(),
|
||||
current,
|
||||
)?;
|
||||
}
|
||||
if let Some(extra) = extra {
|
||||
for (key, value) in extra {
|
||||
match value {
|
||||
Some(value) => {
|
||||
current.insert(key.clone(), value.clone());
|
||||
}
|
||||
None => {
|
||||
current.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if load_default_env {
|
||||
current.insert("FLUXER_SELF_HOSTED".to_owned(), "true".to_owned());
|
||||
}
|
||||
Ok(current)
|
||||
}
|
||||
|
||||
pub fn run(args: &[&str]) -> Result<()> {
|
||||
run_command(args, RunOptions::default()).map(drop)
|
||||
}
|
||||
|
||||
pub fn run_with_env(args: &[&str], env: Vec<(String, Option<String>)>) -> Result<()> {
|
||||
run_command(
|
||||
args,
|
||||
RunOptions {
|
||||
env,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
pub fn run_capture(
|
||||
args: &[&str],
|
||||
env: Vec<(String, Option<String>)>,
|
||||
check: bool,
|
||||
) -> Result<Output> {
|
||||
run_command(
|
||||
args,
|
||||
RunOptions {
|
||||
env,
|
||||
check,
|
||||
capture: true,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RunOptions<'a> {
|
||||
pub cwd: &'a Path,
|
||||
pub env: Vec<(String, Option<String>)>,
|
||||
pub check: bool,
|
||||
pub capture: bool,
|
||||
pub load_default_env: bool,
|
||||
}
|
||||
|
||||
impl Default for RunOptions<'_> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
cwd: ROOT.as_path(),
|
||||
env: Vec::new(),
|
||||
check: true,
|
||||
capture: false,
|
||||
load_default_env: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run_command(args: &[&str], options: RunOptions<'_>) -> Result<Output> {
|
||||
println!("$ {}", format_command(args));
|
||||
let env = merged_env(Some(&options.env), options.load_default_env)?;
|
||||
let mut command = Command::new(args[0]);
|
||||
command
|
||||
.args(&args[1..])
|
||||
.current_dir(options.cwd)
|
||||
.env_clear()
|
||||
.envs(env);
|
||||
if options.capture {
|
||||
command.stdout(Stdio::piped()).stderr(Stdio::piped());
|
||||
let output = command
|
||||
.output()
|
||||
.with_context(|| format!("failed to run {}", format_command(args)))?;
|
||||
let mut printed = Vec::new();
|
||||
printed.extend_from_slice(&output.stdout);
|
||||
printed.extend_from_slice(&output.stderr);
|
||||
let text = String::from_utf8_lossy(&printed);
|
||||
if !text.trim_end().is_empty() {
|
||||
println!("{}", text.trim_end());
|
||||
}
|
||||
if options.check && !output.status.success() {
|
||||
let code = output.status.code().unwrap_or(-1);
|
||||
bail!(
|
||||
"Command failed with exit code {code}: {}",
|
||||
format_command(args)
|
||||
);
|
||||
}
|
||||
return Ok(output);
|
||||
}
|
||||
|
||||
command
|
||||
.stdin(Stdio::inherit())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit());
|
||||
let status = command
|
||||
.status()
|
||||
.with_context(|| format!("failed to run {}", format_command(args)))?;
|
||||
let output = Output {
|
||||
status,
|
||||
stdout: Vec::new(),
|
||||
stderr: Vec::new(),
|
||||
};
|
||||
if options.check && !output.status.success() {
|
||||
let code = output.status.code().unwrap_or(-1);
|
||||
bail!(
|
||||
"Command failed with exit code {code}: {}",
|
||||
format_command(args)
|
||||
);
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> Result<()> {
|
||||
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
|
||||
let mut last_error = None;
|
||||
while Instant::now() < deadline {
|
||||
let address = format!("{host}:{port}");
|
||||
match address
|
||||
.to_socket_addrs()
|
||||
.ok()
|
||||
.and_then(|mut addrs| addrs.next())
|
||||
.map(|addr| TcpStream::connect_timeout(&addr, Duration::from_secs(2)))
|
||||
{
|
||||
Some(Ok(_)) => {
|
||||
println!("{name} is reachable at {host}:{port}");
|
||||
return Ok(());
|
||||
}
|
||||
Some(Err(error)) => last_error = Some(error.to_string()),
|
||||
None => last_error = Some(format!("failed to resolve {address}")),
|
||||
}
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
bail!(
|
||||
"Timed out waiting for {name} at {host}:{port}: {}",
|
||||
last_error.unwrap_or_else(|| "unknown error".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()?;
|
||||
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
|
||||
let mut last_error = None;
|
||||
while Instant::now() < deadline {
|
||||
match client.get(url).send().await {
|
||||
Ok(response) if response.status().as_u16() < 500 => {
|
||||
println!("{name} is reachable at {url}");
|
||||
return Ok(());
|
||||
}
|
||||
Ok(response) => last_error = Some(format!("HTTP {}", response.status())),
|
||||
Err(error) => last_error = Some(error.to_string()),
|
||||
}
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
bail!(
|
||||
"Timed out waiting for {name} at {url}: {}",
|
||||
last_error.unwrap_or_else(|| "unknown error".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
pub const RESTART_WINDOW: Duration = Duration::from_secs(60);
|
||||
pub const RESTART_LIMIT: usize = 5;
|
||||
|
||||
pub fn restart_budget_exceeded(restarts: &mut VecDeque<Instant>, now: Instant) -> bool {
|
||||
while restarts
|
||||
.front()
|
||||
.is_some_and(|at| now.duration_since(*at) > RESTART_WINDOW)
|
||||
{
|
||||
restarts.pop_front();
|
||||
}
|
||||
if restarts.len() >= RESTART_LIMIT {
|
||||
return true;
|
||||
}
|
||||
restarts.push_back(now);
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub struct ShutdownSignal {
|
||||
interrupt: tokio::signal::unix::Signal,
|
||||
terminate: tokio::signal::unix::Signal,
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
impl ShutdownSignal {
|
||||
pub fn new() -> Result<Self> {
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
Ok(Self {
|
||||
interrupt: signal(SignalKind::interrupt())?,
|
||||
terminate: signal(SignalKind::terminate())?,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn recv(&mut self) -> &'static str {
|
||||
tokio::select! {
|
||||
_ = self.interrupt.recv() => "SIGINT",
|
||||
_ = self.terminate.recv() => "SIGTERM",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub struct ShutdownSignal;
|
||||
|
||||
#[cfg(not(unix))]
|
||||
impl ShutdownSignal {
|
||||
pub fn new() -> Result<Self> {
|
||||
Ok(Self)
|
||||
}
|
||||
|
||||
pub async fn recv(&mut self) -> &'static str {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
"signal"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn formats_posix_commands_for_logs() {
|
||||
assert_eq!(format_command(&["pnpm", "build"]), "pnpm build");
|
||||
assert_eq!(
|
||||
format_command(&["", "two words", "a'b"]),
|
||||
"'' 'two words' 'a'\"'\"'b'"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_budget_allows_limited_restarts_within_window() {
|
||||
let mut restarts = VecDeque::new();
|
||||
let base = Instant::now();
|
||||
for offset in 0..RESTART_LIMIT {
|
||||
assert!(!restart_budget_exceeded(
|
||||
&mut restarts,
|
||||
base + Duration::from_secs(offset as u64)
|
||||
));
|
||||
}
|
||||
assert!(restart_budget_exceeded(
|
||||
&mut restarts,
|
||||
base + Duration::from_secs(RESTART_LIMIT as u64)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_budget_resets_after_window_elapses() {
|
||||
let mut restarts = VecDeque::new();
|
||||
let base = Instant::now();
|
||||
for _ in 0..RESTART_LIMIT {
|
||||
assert!(!restart_budget_exceeded(&mut restarts, base));
|
||||
}
|
||||
assert!(!restart_budget_exceeded(
|
||||
&mut restarts,
|
||||
base + RESTART_WINDOW + Duration::from_secs(1)
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::manifest::{ANY_HOST, DEV_PROXY_PORT, LOCAL_APP_URL, PROXY_ROUTES, ProxyRoute};
|
||||
use anyhow::{Context, Result, bail};
|
||||
use axum::{
|
||||
Router,
|
||||
body::Body,
|
||||
extract::{ConnectInfo, State},
|
||||
http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, header},
|
||||
response::IntoResponse,
|
||||
routing::any,
|
||||
};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use url::Url;
|
||||
|
||||
static ROUTE_CURSORS: LazyLock<Mutex<HashMap<&'static str, usize>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ProxyState {
|
||||
http_client: reqwest::Client,
|
||||
}
|
||||
|
||||
const BLOCKED_REQUEST_HEADERS: &[&str] = &[
|
||||
"connection",
|
||||
"content-length",
|
||||
"host",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailer",
|
||||
"trailers",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
];
|
||||
|
||||
const BLOCKED_RESPONSE_HEADERS: &[&str] = &[
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailer",
|
||||
"trailers",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
];
|
||||
const X_FORWARDED_FOR_HEADER: &str = "x-forwarded-for";
|
||||
|
||||
pub async fn run_proxy(host: &str, port: u16) -> Result<()> {
|
||||
let bind = format!("{host}:{port}");
|
||||
let listener = TcpListener::bind(&bind).await?;
|
||||
println!("Fluxer dev proxy listening on {}", listener.local_addr()?);
|
||||
|
||||
let http_client = reqwest::Client::builder()
|
||||
.pool_max_idle_per_host(64)
|
||||
.pool_idle_timeout(Duration::from_secs(90))
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.tcp_nodelay(true)
|
||||
.build()
|
||||
.context("failed to build dev proxy HTTP client")?;
|
||||
let app = Router::new()
|
||||
.fallback(any(proxy_request))
|
||||
.with_state(ProxyState { http_client });
|
||||
|
||||
axum::serve(
|
||||
listener,
|
||||
app.into_make_service_with_connect_info::<SocketAddr>(),
|
||||
)
|
||||
.await
|
||||
.context("dev proxy server exited unexpectedly")
|
||||
}
|
||||
|
||||
async fn proxy_request(
|
||||
State(state): State<ProxyState>,
|
||||
ConnectInfo(client_addr): ConnectInfo<SocketAddr>,
|
||||
request: Request<Body>,
|
||||
) -> Response<Body> {
|
||||
let request_head = request_head_from_request(&request);
|
||||
|
||||
if let Some(location) = tunnel_public_redirect_location(&request_head) {
|
||||
return redirect_response(&request_head, &location).into_response();
|
||||
}
|
||||
|
||||
let route = route_for_path(&request_head.path);
|
||||
if is_upgrade_request(&request_head) {
|
||||
return proxy_upgrade(request, request_head, route, client_addr).await;
|
||||
}
|
||||
|
||||
proxy_http(state, request, request_head, route, client_addr).await
|
||||
}
|
||||
|
||||
async fn proxy_http(
|
||||
state: ProxyState,
|
||||
request: Request<Body>,
|
||||
request_head: RequestHead,
|
||||
route: &'static ProxyRoute,
|
||||
client_addr: SocketAddr,
|
||||
) -> Response<Body> {
|
||||
let (target_host, target_port) = target_for_route(route);
|
||||
let target_url = upstream_http_url(&request_head.path, route, target_host, target_port);
|
||||
let (parts, body) = request.into_parts();
|
||||
let method = parts.method.clone();
|
||||
let mut request_builder = state.http_client.request(method.clone(), &target_url);
|
||||
if method != Method::GET && method != Method::HEAD {
|
||||
request_builder = request_builder.body(reqwest::Body::wrap_stream(body.into_data_stream()));
|
||||
}
|
||||
let mut saw_forwarded_for = false;
|
||||
|
||||
for (name, value) in &parts.headers {
|
||||
let name_str = name.as_str();
|
||||
if BLOCKED_REQUEST_HEADERS.contains(&name_str) {
|
||||
continue;
|
||||
}
|
||||
if name.as_str().eq_ignore_ascii_case(X_FORWARDED_FOR_HEADER) {
|
||||
saw_forwarded_for = true;
|
||||
if let Some(value) = append_forwarded_for(value, client_addr) {
|
||||
request_builder = request_builder.header(name.clone(), value);
|
||||
}
|
||||
} else {
|
||||
request_builder = request_builder.header(name.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
if !saw_forwarded_for {
|
||||
request_builder =
|
||||
request_builder.header(X_FORWARDED_FOR_HEADER, client_addr.ip().to_string());
|
||||
}
|
||||
|
||||
let upstream_response = match request_builder.send().await {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
eprintln!("dev proxy HTTP request failed for {target_url}: {error:#}");
|
||||
return bad_gateway_response(error);
|
||||
}
|
||||
};
|
||||
|
||||
let status = upstream_response.status();
|
||||
let headers = upstream_response.headers().clone();
|
||||
let mut response = Response::new(Body::from_stream(upstream_response.bytes_stream()));
|
||||
*response.status_mut() = status;
|
||||
copy_response_headers(&headers, response.headers_mut());
|
||||
response
|
||||
}
|
||||
|
||||
async fn proxy_upgrade(
|
||||
mut request: Request<Body>,
|
||||
request_head: RequestHead,
|
||||
route: &'static ProxyRoute,
|
||||
client_addr: SocketAddr,
|
||||
) -> Response<Body> {
|
||||
let on_upgrade = hyper::upgrade::on(&mut request);
|
||||
let (target_host, target_port) = target_for_route(route);
|
||||
let mut target = match TcpStream::connect((target_host, target_port)).await {
|
||||
Ok(target) => target,
|
||||
Err(error) => return bad_gateway_response(error),
|
||||
};
|
||||
|
||||
if let Err(error) = target
|
||||
.write_all(&rewrite_head(
|
||||
&request_head,
|
||||
route,
|
||||
Some(target_host),
|
||||
Some(target_port),
|
||||
Some(&client_addr.ip().to_string()),
|
||||
))
|
||||
.await
|
||||
{
|
||||
return bad_gateway_response(error);
|
||||
}
|
||||
|
||||
let (head, body_prefix) = match read_http_head(&mut target).await {
|
||||
Ok(value) => value,
|
||||
Err(error) => return bad_gateway_response(error),
|
||||
};
|
||||
let upstream_response = match parse_response_head(&head) {
|
||||
Ok(value) => value,
|
||||
Err(error) => return bad_gateway_response(error),
|
||||
};
|
||||
|
||||
let mut response = Response::new(Body::empty());
|
||||
*response.status_mut() = upstream_response.status;
|
||||
copy_upgrade_response_headers(&upstream_response.headers, response.headers_mut());
|
||||
|
||||
if upstream_response.status != StatusCode::SWITCHING_PROTOCOLS {
|
||||
*response.body_mut() = Body::from(body_prefix);
|
||||
return response;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let result: Result<()> = async move {
|
||||
let upgraded = on_upgrade.await?;
|
||||
let mut client = TokioIo::new(upgraded);
|
||||
if !body_prefix.is_empty() {
|
||||
client.write_all(&body_prefix).await?;
|
||||
}
|
||||
tokio::io::copy_bidirectional(&mut client, &mut target).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
if let Err(error) = result {
|
||||
eprintln!("dev proxy upgrade bridge failed: {error:#}");
|
||||
}
|
||||
});
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
fn upstream_http_url(
|
||||
request_path: &str,
|
||||
route: &ProxyRoute,
|
||||
target_host: &str,
|
||||
target_port: u16,
|
||||
) -> String {
|
||||
format!(
|
||||
"http://{target_host}:{target_port}{}",
|
||||
rewrite_path(request_path, route)
|
||||
)
|
||||
}
|
||||
|
||||
fn request_head_from_request(request: &Request<Body>) -> RequestHead {
|
||||
RequestHead {
|
||||
method: request.method().as_str().to_owned(),
|
||||
path: request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(|value| value.as_str().to_owned())
|
||||
.unwrap_or_else(|| request.uri().to_string()),
|
||||
version: format!("{:?}", request.version()),
|
||||
headers: request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
(
|
||||
name.as_str().to_owned(),
|
||||
value.to_str().unwrap_or_default().to_owned(),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn append_forwarded_for(value: &HeaderValue, client_addr: SocketAddr) -> Option<HeaderValue> {
|
||||
let value = value.to_str().ok()?;
|
||||
HeaderValue::from_str(&format!("{value}, {}", client_addr.ip())).ok()
|
||||
}
|
||||
|
||||
fn copy_response_headers(source: &HeaderMap, target: &mut HeaderMap) {
|
||||
for (name, value) in source {
|
||||
if BLOCKED_RESPONSE_HEADERS.contains(&name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
target.append(name.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
fn copy_upgrade_response_headers(source: &[(String, String)], target: &mut HeaderMap) {
|
||||
for (name, value) in source {
|
||||
let Ok(name) = axum::http::HeaderName::from_bytes(name.as_bytes()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(value) = HeaderValue::from_str(value) else {
|
||||
continue;
|
||||
};
|
||||
target.append(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
fn bad_gateway_response(error: impl std::fmt::Display) -> Response<Body> {
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
[(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("text/plain; charset=utf-8"),
|
||||
)],
|
||||
format!("{error}\n"),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn read_http_head(stream: &mut TcpStream) -> Result<(Vec<u8>, Vec<u8>)> {
|
||||
let mut head = Vec::new();
|
||||
let mut buf = [0_u8; 4096];
|
||||
loop {
|
||||
let count = stream.read(&mut buf).await?;
|
||||
if count == 0 {
|
||||
bail!("connection closed before response head");
|
||||
}
|
||||
head.extend_from_slice(&buf[..count]);
|
||||
if head.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
if head.len() > 64 * 1024 {
|
||||
bail!("response head exceeded 64KiB");
|
||||
}
|
||||
}
|
||||
let header_end = head
|
||||
.windows(4)
|
||||
.position(|window| window == b"\r\n\r\n")
|
||||
.map(|index| index + 4)
|
||||
.context("missing response header terminator")?;
|
||||
let body_prefix = head.split_off(header_end);
|
||||
Ok((head, body_prefix))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct ResponseHead {
|
||||
status: StatusCode,
|
||||
headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
fn parse_response_head(head: &[u8]) -> Result<ResponseHead> {
|
||||
let text = String::from_utf8_lossy(head);
|
||||
let mut lines = text.split("\r\n");
|
||||
let status_line = lines.next().unwrap_or_default();
|
||||
let parts = status_line.splitn(3, ' ').collect::<Vec<_>>();
|
||||
if parts.len() < 2 {
|
||||
bail!("invalid response status line: {status_line:?}");
|
||||
}
|
||||
let status_code = parts[1]
|
||||
.parse::<u16>()
|
||||
.with_context(|| format!("invalid response status code: {:?}", parts[1]))?;
|
||||
let status = StatusCode::from_u16(status_code)
|
||||
.with_context(|| format!("unsupported response status code: {status_code}"))?;
|
||||
let headers = lines
|
||||
.filter_map(|line| {
|
||||
if line.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (name, value) = line.split_once(':')?;
|
||||
(!name.is_empty()).then(|| (name.to_owned(), value.trim().to_owned()))
|
||||
})
|
||||
.collect();
|
||||
Ok(ResponseHead { status, headers })
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RequestHead {
|
||||
pub method: String,
|
||||
pub path: String,
|
||||
pub version: String,
|
||||
pub headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
pub fn parse_request_head(head: &[u8]) -> Result<RequestHead> {
|
||||
let text = String::from_utf8_lossy(head);
|
||||
let mut lines = text.split("\r\n");
|
||||
let request_line = lines.next().unwrap_or_default();
|
||||
let parts = request_line.splitn(3, ' ').collect::<Vec<_>>();
|
||||
if parts.len() != 3 {
|
||||
bail!("invalid request line: {request_line:?}");
|
||||
}
|
||||
let headers = lines
|
||||
.filter_map(|line| {
|
||||
if line.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (name, value) = line.split_once(':')?;
|
||||
(!name.is_empty()).then(|| (name.to_owned(), value.trim().to_owned()))
|
||||
})
|
||||
.collect();
|
||||
Ok(RequestHead {
|
||||
method: parts[0].to_owned(),
|
||||
path: parts[1].to_owned(),
|
||||
version: parts[2].to_owned(),
|
||||
headers,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn tunnel_public_redirect_location(request: &RequestHead) -> Option<String> {
|
||||
let public_url = env::var("FLUXER_PUBLIC_URL").ok()?;
|
||||
redirect_location_for_public_url(request, &public_url)
|
||||
}
|
||||
|
||||
pub fn redirect_location_for_public_url(request: &RequestHead, public_url: &str) -> Option<String> {
|
||||
if !request.method.eq_ignore_ascii_case("GET") && !request.method.eq_ignore_ascii_case("HEAD") {
|
||||
return None;
|
||||
}
|
||||
if !is_local_dev_host(request) || !accepts_html(request) {
|
||||
return None;
|
||||
}
|
||||
let parsed_public_url = Url::parse(public_url).ok()?;
|
||||
if is_local_url(&parsed_public_url) {
|
||||
return None;
|
||||
}
|
||||
let path = normalize_request_target(&request.path);
|
||||
Some(format!(
|
||||
"{}{}",
|
||||
parsed_public_url.as_str().trim_end_matches('/'),
|
||||
path
|
||||
))
|
||||
}
|
||||
|
||||
fn redirect_response(request: &RequestHead, location: &str) -> Response<Body> {
|
||||
let body = if request.method.eq_ignore_ascii_case("HEAD") {
|
||||
String::new()
|
||||
} else {
|
||||
format!("Redirecting to {location}\n")
|
||||
};
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = StatusCode::FOUND;
|
||||
if let Ok(value) = HeaderValue::from_str(location) {
|
||||
response.headers_mut().insert(header::LOCATION, value);
|
||||
}
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("text/plain; charset=utf-8"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn is_local_dev_host(request: &RequestHead) -> bool {
|
||||
let Some(host) = header_value(request, "host") else {
|
||||
return false;
|
||||
};
|
||||
matches!(
|
||||
host_without_port(host).as_deref(),
|
||||
Some("localhost" | "127.0.0.1" | "::1" | "0.0.0.0")
|
||||
)
|
||||
}
|
||||
|
||||
fn is_local_url(url: &Url) -> bool {
|
||||
if url.as_str().trim_end_matches('/') == LOCAL_APP_URL {
|
||||
return true;
|
||||
}
|
||||
matches!(
|
||||
url.host_str(),
|
||||
Some("localhost" | "127.0.0.1" | "::1" | "0.0.0.0")
|
||||
)
|
||||
}
|
||||
|
||||
fn accepts_html(request: &RequestHead) -> bool {
|
||||
header_value(request, "accept")
|
||||
.map(|value| value.to_ascii_lowercase().contains("text/html"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn is_upgrade_request(request: &RequestHead) -> bool {
|
||||
let connection_upgrade = header_value(request, "connection")
|
||||
.map(|value| value.to_ascii_lowercase().contains("upgrade"))
|
||||
.unwrap_or(false);
|
||||
connection_upgrade && header_value(request, "upgrade").is_some()
|
||||
}
|
||||
|
||||
fn header_value<'a>(request: &'a RequestHead, name: &str) -> Option<&'a str> {
|
||||
request
|
||||
.headers
|
||||
.iter()
|
||||
.find(|(header_name, _)| header_name.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
|
||||
fn host_without_port(host: &str) -> Option<String> {
|
||||
let host = host.trim().to_ascii_lowercase();
|
||||
if host.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if let Some(rest) = host.strip_prefix('[') {
|
||||
return rest.split_once(']').map(|(host, _)| host.to_owned());
|
||||
}
|
||||
Some(host.split(':').next().unwrap_or(&host).to_owned())
|
||||
}
|
||||
|
||||
pub fn route_for_path(path: &str) -> &'static ProxyRoute {
|
||||
let normalized_path = normalize_request_target(path);
|
||||
let parsed_path = Url::parse(&format!("http://fluxer.local{normalized_path}"))
|
||||
.map(|url| url.path().to_owned())
|
||||
.unwrap_or_else(|_| {
|
||||
normalized_path
|
||||
.split('?')
|
||||
.next()
|
||||
.unwrap_or(&normalized_path)
|
||||
.to_owned()
|
||||
});
|
||||
for route in PROXY_ROUTES {
|
||||
if route.prefix == "/" {
|
||||
return route;
|
||||
}
|
||||
if parsed_path == route.prefix
|
||||
|| parsed_path.starts_with(&format!("{}/", route.prefix))
|
||||
|| (route.prefix.ends_with('-') && parsed_path.starts_with(route.prefix))
|
||||
{
|
||||
return route;
|
||||
}
|
||||
}
|
||||
PROXY_ROUTES.last().expect("proxy has fallback route")
|
||||
}
|
||||
|
||||
pub fn target_for_route(route: &'static ProxyRoute) -> (&'static str, u16) {
|
||||
if route.alternate_ports.is_empty() {
|
||||
return (route.host, route.port);
|
||||
}
|
||||
let ports = std::iter::once(route.port)
|
||||
.chain(route.alternate_ports.iter().copied())
|
||||
.collect::<Vec<_>>();
|
||||
let mut cursors = ROUTE_CURSORS.lock().expect("route cursor lock poisoned");
|
||||
let cursor = *cursors.get(route.prefix).unwrap_or(&0);
|
||||
cursors.insert(route.prefix, (cursor + 1) % ports.len());
|
||||
(route.host, ports[cursor])
|
||||
}
|
||||
|
||||
pub fn rewrite_path(path: &str, route: &ProxyRoute) -> String {
|
||||
let normalized_path = normalize_request_target(path);
|
||||
if !route.strip_prefix {
|
||||
return normalized_path;
|
||||
}
|
||||
let (parsed_path, query) = split_path_query(&normalized_path);
|
||||
let next_path = parsed_path
|
||||
.strip_prefix(route.prefix)
|
||||
.unwrap_or(parsed_path);
|
||||
let next_path = if next_path.is_empty() { "/" } else { next_path };
|
||||
match query {
|
||||
Some(query) => format!("{next_path}?{query}"),
|
||||
None => next_path.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_request_target(request_target: &str) -> String {
|
||||
let Ok(url) = Url::parse(request_target) else {
|
||||
return request_target.to_owned();
|
||||
};
|
||||
if url.scheme() != "http" && url.scheme() != "https" {
|
||||
return request_target.to_owned();
|
||||
}
|
||||
let mut path = url.path().to_owned();
|
||||
if path.is_empty() {
|
||||
path.push('/');
|
||||
}
|
||||
if let Some(query) = url.query() {
|
||||
path.push('?');
|
||||
path.push_str(query);
|
||||
}
|
||||
path
|
||||
}
|
||||
|
||||
fn split_path_query(path: &str) -> (&str, Option<&str>) {
|
||||
path.split_once('?')
|
||||
.map(|(path, query)| (path, Some(query)))
|
||||
.unwrap_or((path, None))
|
||||
}
|
||||
|
||||
pub fn rewrite_head(
|
||||
request: &RequestHead,
|
||||
route: &ProxyRoute,
|
||||
target_host: Option<&str>,
|
||||
target_port: Option<u16>,
|
||||
client_ip: Option<&str>,
|
||||
) -> Vec<u8> {
|
||||
let host = target_host.unwrap_or(route.host);
|
||||
let port = target_port.unwrap_or(route.port);
|
||||
let is_upgrade = request
|
||||
.headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("upgrade"));
|
||||
let mut lines = vec![format!(
|
||||
"{} {} {}",
|
||||
request.method,
|
||||
rewrite_path(&request.path, route),
|
||||
request.version
|
||||
)];
|
||||
let mut saw_forwarded_for = false;
|
||||
let mut saw_host = false;
|
||||
let mut saw_connection = false;
|
||||
for (name, value) in &request.headers {
|
||||
if name.eq_ignore_ascii_case("host") {
|
||||
saw_host = true;
|
||||
lines.push(format!("Host: {host}:{port}"));
|
||||
} else if name.eq_ignore_ascii_case("connection") {
|
||||
saw_connection = true;
|
||||
lines.push(format!(
|
||||
"Connection: {}",
|
||||
if is_upgrade { "Upgrade" } else { "close" }
|
||||
));
|
||||
} else if name.eq_ignore_ascii_case("x-forwarded-for") {
|
||||
saw_forwarded_for = true;
|
||||
if let Some(client_ip) = client_ip {
|
||||
lines.push(format!("{name}: {value}, {client_ip}"));
|
||||
} else {
|
||||
lines.push(format!("{name}: {value}"));
|
||||
}
|
||||
} else {
|
||||
lines.push(format!("{name}: {value}"));
|
||||
}
|
||||
}
|
||||
if !saw_host {
|
||||
lines.push(format!("Host: {host}:{port}"));
|
||||
}
|
||||
if !saw_connection {
|
||||
lines.push(format!(
|
||||
"Connection: {}",
|
||||
if is_upgrade { "Upgrade" } else { "close" }
|
||||
));
|
||||
}
|
||||
if let Some(client_ip) = client_ip
|
||||
&& !saw_forwarded_for
|
||||
{
|
||||
lines.push(format!("X-Forwarded-For: {client_ip}"));
|
||||
}
|
||||
lines.push(String::new());
|
||||
lines.push(String::new());
|
||||
lines.join("\r\n").into_bytes()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn default_bind() -> (&'static str, u16) {
|
||||
(ANY_HOST, DEV_PROXY_PORT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn routes_and_rewrites_prefixes() {
|
||||
let api = route_for_path("/api/users?x=1");
|
||||
assert_eq!(api.prefix, "/api");
|
||||
assert_eq!(rewrite_path("/api/users?x=1", api), "/users?x=1");
|
||||
let asset = route_for_path("/assets/app.js");
|
||||
assert_eq!(rewrite_path("/assets/app.js", asset), "/assets/app.js");
|
||||
let lazy = route_for_path("/lazy-compilation-using-foo");
|
||||
assert_eq!(lazy.prefix, "/lazy-compilation-using-");
|
||||
let devmail = route_for_path("/devmail/messages");
|
||||
assert_eq!(devmail.prefix, "/devmail");
|
||||
assert_eq!(
|
||||
rewrite_path("/devmail/messages?x=1", devmail),
|
||||
"/devmail/messages?x=1"
|
||||
);
|
||||
let admin = route_for_path("/admin/users");
|
||||
assert_eq!(admin.prefix, "/admin");
|
||||
assert_eq!(rewrite_path("/admin/users", admin), "/users");
|
||||
let absolute_gateway = route_for_path("https://dev.example.test/gateway?v=1&encoding=json");
|
||||
assert_eq!(absolute_gateway.prefix, "/gateway");
|
||||
assert_eq!(
|
||||
rewrite_path(
|
||||
"https://dev.example.test/gateway?v=1&encoding=json",
|
||||
absolute_gateway
|
||||
),
|
||||
"/?v=1&encoding=json"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_host_connection_and_forwarded_for() {
|
||||
let request = RequestHead {
|
||||
method: "GET".to_owned(),
|
||||
path: "/api/_health".to_owned(),
|
||||
version: "HTTP/1.1".to_owned(),
|
||||
headers: vec![
|
||||
("Host".to_owned(), "localhost:8088".to_owned()),
|
||||
("Connection".to_owned(), "keep-alive".to_owned()),
|
||||
("X-Forwarded-For".to_owned(), "1.1.1.1".to_owned()),
|
||||
],
|
||||
};
|
||||
let head = String::from_utf8(rewrite_head(
|
||||
&request,
|
||||
route_for_path("/api/_health"),
|
||||
Some("api"),
|
||||
Some(8080),
|
||||
Some("2.2.2.2"),
|
||||
))
|
||||
.unwrap();
|
||||
assert!(head.contains("GET /_health HTTP/1.1\r\n"));
|
||||
assert!(head.contains("Host: api:8080\r\n"));
|
||||
assert!(head.contains("Connection: close\r\n"));
|
||||
assert!(head.contains("X-Forwarded-For: 1.1.1.1, 2.2.2.2\r\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redirects_local_html_navigation_to_public_tunnel_url() {
|
||||
let request = RequestHead {
|
||||
method: "GET".to_owned(),
|
||||
path: "/channels/@me?x=1".to_owned(),
|
||||
version: "HTTP/1.1".to_owned(),
|
||||
headers: vec![
|
||||
("Host".to_owned(), "localhost:8088".to_owned()),
|
||||
(
|
||||
"Accept".to_owned(),
|
||||
"text/html,application/xhtml+xml".to_owned(),
|
||||
),
|
||||
],
|
||||
};
|
||||
assert_eq!(
|
||||
redirect_location_for_public_url(&request, "https://dev.example.test"),
|
||||
Some("https://dev.example.test/channels/@me?x=1".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_tunnel_redirect_ignores_assets_and_local_public_url() {
|
||||
let asset = RequestHead {
|
||||
method: "GET".to_owned(),
|
||||
path: "/avatars/4.png".to_owned(),
|
||||
version: "HTTP/1.1".to_owned(),
|
||||
headers: vec![
|
||||
("Host".to_owned(), "localhost:8088".to_owned()),
|
||||
(
|
||||
"Accept".to_owned(),
|
||||
"image/avif,image/webp,image/apng,image/*,*/*;q=0.8".to_owned(),
|
||||
),
|
||||
],
|
||||
};
|
||||
assert_eq!(
|
||||
redirect_location_for_public_url(&asset, "https://dev.example.test"),
|
||||
None
|
||||
);
|
||||
|
||||
let local_public_url = RequestHead {
|
||||
method: "GET".to_owned(),
|
||||
path: "/channels/@me".to_owned(),
|
||||
version: "HTTP/1.1".to_owned(),
|
||||
headers: vec![
|
||||
("Host".to_owned(), "localhost:8088".to_owned()),
|
||||
("Accept".to_owned(), "text/html".to_owned()),
|
||||
],
|
||||
};
|
||||
assert_eq!(
|
||||
redirect_location_for_public_url(&local_public_url, LOCAL_APP_URL),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::manifest::{DEV_PROXY_PORT, MEDIA_PROXY_PORT, RustServiceSpec, rust_services};
|
||||
use crate::paths::ROOT;
|
||||
use crate::proc::{
|
||||
RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,
|
||||
restart_budget_exceeded, run_command,
|
||||
};
|
||||
use anyhow::{Result, bail};
|
||||
use std::collections::{BTreeSet, VecDeque};
|
||||
use std::env;
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::fs;
|
||||
use std::net::{SocketAddr, TcpListener};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
|
||||
const SERVICE_RESTART_PORT_WAIT: Duration = Duration::from_secs(75);
|
||||
|
||||
struct SupervisedService {
|
||||
spec: RustServiceSpec,
|
||||
mode: &'static str,
|
||||
port: u16,
|
||||
child: Child,
|
||||
restarts: VecDeque<Instant>,
|
||||
}
|
||||
|
||||
impl SupervisedService {
|
||||
fn label(&self) -> String {
|
||||
format!("{}:{}", self.spec.name, self.mode)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_rust_services(service_names: &[String]) -> Result<i32> {
|
||||
let selected = select_services(service_names)?;
|
||||
cleanup_orphaned_service_processes(&selected).await?;
|
||||
wait_for_service_ports_available(&selected)?;
|
||||
build_services(&selected)?;
|
||||
let mut shutdown = ShutdownSignal::new()?;
|
||||
let mut supervised = Vec::new();
|
||||
for spec in &selected {
|
||||
for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {
|
||||
supervised.push(SupervisedService {
|
||||
spec: spec.clone(),
|
||||
mode,
|
||||
port,
|
||||
child: start_service(spec, mode, port)?,
|
||||
restarts: VecDeque::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
loop {
|
||||
if let Err(error) = restart_exited_services(&mut supervised).await {
|
||||
stop_supervised_services(&mut supervised);
|
||||
return Err(error);
|
||||
}
|
||||
tokio::select! {
|
||||
signal = shutdown.recv() => {
|
||||
println!("Received {signal}; stopping Rust service tasks...");
|
||||
stop_supervised_services(&mut supervised);
|
||||
return Ok(0);
|
||||
}
|
||||
_ = sleep(Duration::from_millis(500)) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn restart_exited_services(supervised: &mut [SupervisedService]) -> Result<()> {
|
||||
for entry in supervised.iter_mut() {
|
||||
let Some(status) = entry.child.try_wait()? else {
|
||||
continue;
|
||||
};
|
||||
if restart_budget_exceeded(&mut entry.restarts, Instant::now()) {
|
||||
bail!(
|
||||
"Rust service {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",
|
||||
entry.label(),
|
||||
RESTART_WINDOW.as_secs()
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"[{}] exited with {status}; restarting service",
|
||||
entry.label()
|
||||
);
|
||||
wait_for_port_available(&entry.label(), entry.port).await?;
|
||||
entry.child = start_service(&entry.spec, entry.mode, entry.port)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_port_available(label: &str, port: u16) -> Result<()> {
|
||||
let deadline = Instant::now() + SERVICE_RESTART_PORT_WAIT;
|
||||
loop {
|
||||
if can_bind_tcp_port(port) {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
bail!("port {port} for Rust service {label} did not become available");
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_supervised_services(supervised: &mut [SupervisedService]) {
|
||||
let mut children = supervised
|
||||
.iter_mut()
|
||||
.map(|entry| &mut entry.child)
|
||||
.collect::<Vec<_>>();
|
||||
let mut processes = Vec::new();
|
||||
for child in children.drain(..) {
|
||||
processes.push(child);
|
||||
}
|
||||
crate::gateway::stop_child_processes(&mut processes);
|
||||
}
|
||||
|
||||
pub fn select_services(service_names: &[String]) -> Result<Vec<RustServiceSpec>> {
|
||||
let services = rust_services();
|
||||
if service_names.is_empty() {
|
||||
return Ok(services);
|
||||
}
|
||||
let unknown = service_names
|
||||
.iter()
|
||||
.filter(|name| !services.iter().any(|spec| spec.name == name.as_str()))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if !unknown.is_empty() {
|
||||
let available = services
|
||||
.iter()
|
||||
.map(|spec| spec.name)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
bail!(
|
||||
"Unknown Rust service(s): {}. Available: {available}",
|
||||
unknown.join(", ")
|
||||
);
|
||||
}
|
||||
Ok(service_names
|
||||
.iter()
|
||||
.filter_map(|name| {
|
||||
services
|
||||
.iter()
|
||||
.find(|spec| spec.name == name.as_str())
|
||||
.cloned()
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn build_services(services: &[RustServiceSpec]) -> Result<()> {
|
||||
let mut packages = Vec::new();
|
||||
let mut seen = BTreeSet::new();
|
||||
for spec in services {
|
||||
if seen.insert(spec.package) {
|
||||
packages.extend(["-p".to_owned(), spec.package.to_owned()]);
|
||||
}
|
||||
}
|
||||
let mut args = vec!["cargo".to_owned(), "build".to_owned()];
|
||||
args.extend(packages);
|
||||
let refs = args.iter().map(String::as_str).collect::<Vec<_>>();
|
||||
run_command(&refs, RunOptions::default()).map(drop)
|
||||
}
|
||||
|
||||
fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child> {
|
||||
let args = service_command(spec);
|
||||
let env = merged_env(Some(&service_env(spec, mode, port)), true)?;
|
||||
let label = format!("{}:{mode}", spec.name);
|
||||
println!("[{label}] $ {}", format_command(&args));
|
||||
let mut command = Command::new(&args[0]);
|
||||
command
|
||||
.args(&args[1..])
|
||||
.current_dir(ROOT.as_path())
|
||||
.env_clear()
|
||||
.envs(env)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
#[cfg(unix)]
|
||||
unsafe {
|
||||
use std::os::unix::process::CommandExt;
|
||||
command.pre_exec(|| {
|
||||
libc::setsid();
|
||||
Ok(())
|
||||
});
|
||||
}
|
||||
let mut child = command.spawn()?;
|
||||
if let Some(stdout) = child.stdout.take() {
|
||||
let stdout_label = label.clone();
|
||||
std::thread::spawn(move || prefix_output(&stdout_label, stdout));
|
||||
}
|
||||
if let Some(stderr) = child.stderr.take() {
|
||||
std::thread::spawn(move || prefix_output(&label, stderr));
|
||||
}
|
||||
Ok(child)
|
||||
}
|
||||
|
||||
pub fn service_command(spec: &RustServiceSpec) -> Vec<String> {
|
||||
if env::var("FLUXER_DEV_RUST_HOT_RELOAD")
|
||||
.map(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true" | "yes"))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
let rel = spec.path.strip_prefix(ROOT.as_path()).unwrap_or(&spec.path);
|
||||
return vec![
|
||||
"cargo".to_owned(),
|
||||
"watch".to_owned(),
|
||||
"-w".to_owned(),
|
||||
rel.join("src").display().to_string(),
|
||||
"-w".to_owned(),
|
||||
rel.join("Cargo.toml").display().to_string(),
|
||||
"-w".to_owned(),
|
||||
"fluxer_svc/src".to_owned(),
|
||||
"-w".to_owned(),
|
||||
"fluxer_svc/Cargo.toml".to_owned(),
|
||||
"-x".to_owned(),
|
||||
format!("run -p {}", spec.package),
|
||||
];
|
||||
}
|
||||
vec![
|
||||
"cargo".to_owned(),
|
||||
"run".to_owned(),
|
||||
"-p".to_owned(),
|
||||
spec.package.to_owned(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String, Option<String>)> {
|
||||
let mut envs = vec![
|
||||
("FLUXER_SVC_NAME".to_owned(), Some(spec.name.to_owned())),
|
||||
("FLUXER_SVC_MODE".to_owned(), Some(mode.to_owned())),
|
||||
("FLUXER_SVC_PORT".to_owned(), Some(port.to_string())),
|
||||
(
|
||||
"FLUXER_SVC_LISTEN_HOST".to_owned(),
|
||||
Some("0.0.0.0".to_owned()),
|
||||
),
|
||||
("FLUXER_SVC_SHARD_COUNT".to_owned(), Some("1".to_owned())),
|
||||
(
|
||||
"FLUXER_SVC_NATS_URL".to_owned(),
|
||||
Some(
|
||||
env::var("FLUXER_SVC_NATS_URL")
|
||||
.or_else(|_| env::var("FLUXER_NATS_URL"))
|
||||
.unwrap_or_else(|_| "nats://nats:4222".to_owned()),
|
||||
),
|
||||
),
|
||||
(
|
||||
"FLUXER_CASSANDRA_HOSTS".to_owned(),
|
||||
Some(env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned())),
|
||||
),
|
||||
(
|
||||
"FLUXER_CASSANDRA_KEYSPACE".to_owned(),
|
||||
Some(env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned())),
|
||||
),
|
||||
(
|
||||
"FLUXER_CASSANDRA_PORT".to_owned(),
|
||||
Some(env::var("FLUXER_CASSANDRA_PORT").unwrap_or_else(|_| "9042".to_owned())),
|
||||
),
|
||||
];
|
||||
if mode == "shard" {
|
||||
envs.push(("FLUXER_SVC_SHARD_ID".to_owned(), Some("0".to_owned())));
|
||||
}
|
||||
if spec.name == "unfurl" {
|
||||
envs.extend([
|
||||
(
|
||||
"FLUXER_MEDIA_PROXY_ENDPOINT".to_owned(),
|
||||
Some(
|
||||
env::var("FLUXER_MEDIA_PROXY_ENDPOINT")
|
||||
.unwrap_or_else(|_| format!("http://127.0.0.1:{MEDIA_PROXY_PORT}")),
|
||||
),
|
||||
),
|
||||
(
|
||||
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT".to_owned(),
|
||||
Some(
|
||||
env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
|
||||
.or_else(|_| env::var("FLUXER_MEDIA_ENDPOINT"))
|
||||
.unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}/media")),
|
||||
),
|
||||
),
|
||||
(
|
||||
"FLUXER_STATIC_CDN_ENDPOINT".to_owned(),
|
||||
Some(
|
||||
env::var("FLUXER_STATIC_CDN_ENDPOINT")
|
||||
.or_else(|_| env::var("FLUXER_PUBLIC_URL"))
|
||||
.unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}")),
|
||||
),
|
||||
),
|
||||
]);
|
||||
}
|
||||
envs
|
||||
}
|
||||
|
||||
fn prefix_output(label: &str, reader: impl std::io::Read) {
|
||||
use std::io::{BufRead, BufReader};
|
||||
for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
|
||||
println!("[{label}] {line}");
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_for_service_ports_available(services: &[RustServiceSpec]) -> Result<()> {
|
||||
let conflicts = services
|
||||
.iter()
|
||||
.flat_map(|spec| {
|
||||
[
|
||||
(
|
||||
!can_bind_tcp_port(spec.port_base),
|
||||
format!("{}:router={}", spec.name, spec.port_base),
|
||||
),
|
||||
(
|
||||
!can_bind_tcp_port(spec.port_base + 1),
|
||||
format!("{}:shard={}", spec.name, spec.port_base + 1),
|
||||
),
|
||||
]
|
||||
})
|
||||
.filter_map(|(conflict, label)| conflict.then_some(label))
|
||||
.collect::<Vec<_>>();
|
||||
if !conflicts.is_empty() {
|
||||
bail!(
|
||||
"Rust service port(s) already in use: {}. Stop the conflicting process before starting rust-services.",
|
||||
conflicts.join(", ")
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn can_bind_tcp_port(port: u16) -> bool {
|
||||
TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], port))).is_ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Result<()> {
|
||||
let leaders = orphaned_service_leaders(services)?;
|
||||
if leaders.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let pids = leaders.iter().map(|leader| leader.pid).collect::<Vec<_>>();
|
||||
|
||||
println!(
|
||||
"Stopping orphaned Rust service process group(s): {}",
|
||||
format_pids(&pids)
|
||||
);
|
||||
let term_failed = signal_process_groups(&pids, libc::SIGTERM);
|
||||
if !term_failed.is_empty() {
|
||||
println!(
|
||||
"SIGTERM delivery failed for orphaned Rust service pid(s): {}",
|
||||
format_pids(&term_failed)
|
||||
);
|
||||
}
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let remaining = surviving_service_group_pids(&leaders)?;
|
||||
if remaining.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
let survivors = surviving_service_group_pids(&leaders)?;
|
||||
if survivors.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let delivery_failed = kill_failed
|
||||
.into_iter()
|
||||
.filter(|pid| survivors.contains(pid))
|
||||
.collect::<Vec<_>>();
|
||||
if delivery_failed.is_empty() {
|
||||
bail!(
|
||||
"orphaned Rust service process(es) survived SIGKILL: {}",
|
||||
format_pids(&survivors)
|
||||
);
|
||||
}
|
||||
bail!(
|
||||
"orphaned Rust service process(es) survived SIGKILL: {} (signal delivery failed for: {})",
|
||||
format_pids(&survivors),
|
||||
format_pids(&delivery_failed)
|
||||
);
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
async fn cleanup_orphaned_service_processes(_services: &[RustServiceSpec]) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct RustServiceLeader {
|
||||
pid: i32,
|
||||
starttime: u64,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServiceLeader>> {
|
||||
let binaries = services
|
||||
.iter()
|
||||
.map(|spec| format!("target/debug/{}", spec.package))
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mut leaders = Vec::new();
|
||||
for entry in fs::read_dir("/proc")? {
|
||||
let Ok(entry) = entry else {
|
||||
continue;
|
||||
};
|
||||
let Some(pid) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| name.parse::<i32>().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if proc_parent_pid(pid) != Some(1) {
|
||||
continue;
|
||||
}
|
||||
if !cmdline_has_service_binary(&proc_cmdline(pid), &binaries) {
|
||||
continue;
|
||||
}
|
||||
if let Some(starttime) = proc_stat_starttime(pid) {
|
||||
leaders.push(RustServiceLeader { pid, starttime });
|
||||
}
|
||||
}
|
||||
leaders.sort_unstable_by_key(|leader| leader.pid);
|
||||
Ok(leaders)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn format_pids(pids: &[i32]) -> String {
|
||||
if pids.is_empty() {
|
||||
return "none".to_owned();
|
||||
}
|
||||
pids.iter()
|
||||
.map(i32::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
fn cmdline_has_service_binary(args: &[String], binaries: &BTreeSet<String>) -> bool {
|
||||
args.first()
|
||||
.map(|arg| binaries.iter().any(|binary| arg.ends_with(binary)))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn proc_cmdline(pid: i32) -> Vec<String> {
|
||||
fs::read(format!("/proc/{pid}/cmdline"))
|
||||
.unwrap_or_default()
|
||||
.split(|byte| *byte == 0)
|
||||
.filter(|arg| !arg.is_empty())
|
||||
.map(|arg| String::from_utf8_lossy(arg).into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn proc_parent_pid(pid: i32) -> Option<i32> {
|
||||
fs::read_to_string(format!("/proc/{pid}/status"))
|
||||
.ok()?
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn process_exists(pid: i32) -> bool {
|
||||
assert!(pid > 0);
|
||||
match proc_stat_state_and_pgid(pid) {
|
||||
Some((state, _pgid)) => !proc_stat_state_is_dead(state),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {
|
||||
assert!(pid > 0);
|
||||
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
|
||||
parse_proc_stat_state_and_pgid(&stat)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
|
||||
let (_, after_comm) = stat.rsplit_once(')')?;
|
||||
let mut fields = after_comm.split_ascii_whitespace();
|
||||
let state = fields.next()?.chars().next()?;
|
||||
let _ppid: i32 = fields.next()?.parse().ok()?;
|
||||
let pgid: i32 = fields.next()?.parse().ok()?;
|
||||
Some((state, pgid))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn proc_stat_starttime(pid: i32) -> Option<u64> {
|
||||
assert!(pid > 0);
|
||||
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
|
||||
parse_proc_stat_starttime(&stat)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
fn parse_proc_stat_starttime(stat: &str) -> Option<u64> {
|
||||
let (_, after_comm) = stat.rsplit_once(')')?;
|
||||
after_comm.split_ascii_whitespace().nth(19)?.parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
fn proc_stat_state_is_dead(state: char) -> bool {
|
||||
state == 'Z' || state == 'X' || state == 'x'
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn surviving_service_group_pids(leaders: &[RustServiceLeader]) -> Result<Vec<i32>> {
|
||||
let leader_pids = leaders
|
||||
.iter()
|
||||
.map(|leader| leader.pid)
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
let mut survivors = Vec::new();
|
||||
for entry in fs::read_dir("/proc")? {
|
||||
let Ok(entry) = entry else {
|
||||
continue;
|
||||
};
|
||||
let Some(pid) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| name.parse::<i32>().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
|
||||
continue;
|
||||
};
|
||||
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
|
||||
continue;
|
||||
};
|
||||
if proc_stat_state_is_dead(state) {
|
||||
continue;
|
||||
}
|
||||
if leader_pids.contains(&pgid) {
|
||||
survivors.push(pid);
|
||||
continue;
|
||||
}
|
||||
let leader = leaders.iter().find(|leader| leader.pid == pid);
|
||||
if let Some(leader) = leader
|
||||
&& parse_proc_stat_starttime(&stat) == Some(leader.starttime)
|
||||
{
|
||||
survivors.push(pid);
|
||||
}
|
||||
}
|
||||
survivors.sort_unstable();
|
||||
Ok(survivors)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {
|
||||
assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);
|
||||
let mut failed = Vec::with_capacity(pids.len());
|
||||
for pid in pids {
|
||||
assert!(*pid > 0);
|
||||
let group_result = unsafe { libc::kill(-pid, signal) };
|
||||
if group_result == 0 {
|
||||
continue;
|
||||
}
|
||||
let direct_result = unsafe { libc::kill(*pid, signal) };
|
||||
if direct_result == 0 {
|
||||
continue;
|
||||
}
|
||||
if process_exists(*pid) {
|
||||
failed.push(*pid);
|
||||
}
|
||||
}
|
||||
failed
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
fn with_public_env(
|
||||
overrides: &[(&str, Option<&str>)],
|
||||
assertions: impl FnOnce(Vec<(String, Option<String>)>),
|
||||
) {
|
||||
let _guard = ENV_LOCK.lock().unwrap();
|
||||
let keys = [
|
||||
"FLUXER_MEDIA_PROXY_ENDPOINT",
|
||||
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
|
||||
"FLUXER_MEDIA_ENDPOINT",
|
||||
"FLUXER_STATIC_CDN_ENDPOINT",
|
||||
"FLUXER_PUBLIC_URL",
|
||||
];
|
||||
let saved = keys
|
||||
.iter()
|
||||
.map(|key| (*key, env::var(key).ok()))
|
||||
.collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
unsafe {
|
||||
env::remove_var(key);
|
||||
}
|
||||
}
|
||||
for (key, value) in overrides {
|
||||
if let Some(value) = value {
|
||||
unsafe {
|
||||
env::set_var(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let spec = rust_services()
|
||||
.into_iter()
|
||||
.find(|spec| spec.name == "unfurl")
|
||||
.unwrap();
|
||||
let env = service_env(&spec, "router", spec.port_base);
|
||||
|
||||
for (key, value) in saved {
|
||||
match value {
|
||||
Some(value) => unsafe {
|
||||
env::set_var(key, value);
|
||||
},
|
||||
None => unsafe {
|
||||
env::remove_var(key);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
assertions(env);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_services() {
|
||||
let err = select_services(&["bogus".to_owned()])
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("Unknown Rust service"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_unfurl_env_with_media_endpoints() {
|
||||
with_public_env(&[], |env| {
|
||||
assert!(
|
||||
env.iter()
|
||||
.any(|(key, value)| key == "FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT"
|
||||
&& value.as_deref() == Some("http://localhost:8088/media"))
|
||||
);
|
||||
assert!(
|
||||
env.iter()
|
||||
.any(|(key, value)| key == "FLUXER_STATIC_CDN_ENDPOINT"
|
||||
&& value.as_deref() == Some("http://localhost:8088"))
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unfurl_env_uses_public_url_overrides() {
|
||||
with_public_env(
|
||||
&[
|
||||
(
|
||||
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
|
||||
Some("https://dev.example.com/media"),
|
||||
),
|
||||
(
|
||||
"FLUXER_STATIC_CDN_ENDPOINT",
|
||||
Some("https://dev.example.com"),
|
||||
),
|
||||
],
|
||||
|env| {
|
||||
assert!(
|
||||
env.iter()
|
||||
.any(|(key, value)| key == "FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT"
|
||||
&& value.as_deref() == Some("https://dev.example.com/media"))
|
||||
);
|
||||
assert!(
|
||||
env.iter()
|
||||
.any(|(key, value)| key == "FLUXER_STATIC_CDN_ENDPOINT"
|
||||
&& value.as_deref() == Some("https://dev.example.com"))
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cmdline_binary_match_uses_selected_service_binary_suffix() {
|
||||
let binaries = BTreeSet::from(["target/debug/fluxer_messages".to_owned()]);
|
||||
assert!(cmdline_has_service_binary(
|
||||
&["/workspaces/fluxer/target/debug/fluxer_messages".to_owned()],
|
||||
&binaries
|
||||
));
|
||||
assert!(!cmdline_has_service_binary(
|
||||
&["/workspaces/fluxer/target/debug/fluxer_users".to_owned()],
|
||||
&binaries
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proc_stat_parsing_handles_parentheses_and_spaces_in_comm() {
|
||||
assert_eq!(
|
||||
parse_proc_stat_state_and_pgid("99 (spaced comm) T 1 99 99 0 -1"),
|
||||
Some(('T', 99))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_proc_stat_state_and_pgid("99 (weird) comm (name) Z 1 42 42 0 -1"),
|
||||
Some(('Z', 42))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proc_stat_parsing_rejects_malformed_lines() {
|
||||
assert_eq!(parse_proc_stat_state_and_pgid(""), None);
|
||||
assert_eq!(parse_proc_stat_state_and_pgid("1234 (svc)"), None);
|
||||
assert_eq!(parse_proc_stat_state_and_pgid("1234 (svc) S"), None);
|
||||
assert_eq!(parse_proc_stat_state_and_pgid("1234 (svc) S 1"), None);
|
||||
assert_eq!(
|
||||
parse_proc_stat_state_and_pgid("1234 (svc) S 1 not-a-pgid"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proc_stat_parsing_extracts_starttime() {
|
||||
let stat = "1234 (svc) S 1 1234 1234 0 -1 4194560 0 0 0 0 5 3 0 0 20 0 30 0 12345678 4096";
|
||||
assert_eq!(parse_proc_stat_starttime(stat), Some(12_345_678));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proc_stat_starttime_parsing_rejects_truncated_lines() {
|
||||
assert_eq!(parse_proc_stat_starttime(""), None);
|
||||
assert_eq!(parse_proc_stat_starttime("1234 (svc)"), None);
|
||||
assert_eq!(
|
||||
parse_proc_stat_starttime("1234 (svc) S 1 1234 1234 0 -1 4194560 0"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zombie_and_reaped_states_count_as_dead() {
|
||||
assert!(proc_stat_state_is_dead('Z'));
|
||||
assert!(proc_stat_state_is_dead('X'));
|
||||
assert!(proc_stat_state_is_dead('x'));
|
||||
assert!(!proc_stat_state_is_dead('S'));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::cassandra::{apply_schema, config_from_env, verify_schema};
|
||||
use crate::manifest::{API_PORT, DEV_PROXY_PORT, LOOPBACK_HOST};
|
||||
use crate::proc::{RunOptions, merged_env, run_command, wait_http, wait_tcp};
|
||||
use anyhow::{Context, Result, bail};
|
||||
use futures_util::StreamExt;
|
||||
use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderValue};
|
||||
use serde_json::json;
|
||||
use std::env;
|
||||
use std::process::{Command, Output, Stdio};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
use tokio_tungstenite::{connect_async, tungstenite::Message};
|
||||
use url::Url;
|
||||
|
||||
pub const S3_BUCKETS: &[&str] = &[
|
||||
"fluxer",
|
||||
"fluxer-uploads",
|
||||
"fluxer-downloads",
|
||||
"fluxer-reports",
|
||||
"fluxer-harvests",
|
||||
"fluxer-static",
|
||||
];
|
||||
|
||||
pub async fn run_smoke(quick: bool, public: bool) -> Result<()> {
|
||||
let timeout = if quick { 5 } else { 120 };
|
||||
wait_tcp(
|
||||
"Valkey",
|
||||
&env::var("VALKEY_HOST").unwrap_or_else(|_| "valkey".to_owned()),
|
||||
env::var("VALKEY_PORT")
|
||||
.unwrap_or_else(|_| "6379".to_owned())
|
||||
.parse()?,
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
wait_tcp(
|
||||
"NATS",
|
||||
&env::var("NATS_HOST").unwrap_or_else(|_| "nats".to_owned()),
|
||||
env::var("NATS_PORT")
|
||||
.unwrap_or_else(|_| "4222".to_owned())
|
||||
.parse()?,
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
wait_tcp(
|
||||
"LiveKit",
|
||||
&env::var("LIVEKIT_HOST").unwrap_or_else(|_| "livekit".to_owned()),
|
||||
env::var("LIVEKIT_PORT")
|
||||
.unwrap_or_else(|_| "7880".to_owned())
|
||||
.parse()?,
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
wait_tcp(
|
||||
"Mailpit SMTP",
|
||||
&env::var("MAILPIT_HOST").unwrap_or_else(|_| "mailpit".to_owned()),
|
||||
env::var("MAILPIT_SMTP_PORT")
|
||||
.unwrap_or_else(|_| "1025".to_owned())
|
||||
.parse()?,
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
wait_tcp(
|
||||
"SeaweedFS S3",
|
||||
&env::var("S3_HOST").unwrap_or_else(|_| "127.0.0.1".to_owned()),
|
||||
env::var("S3_PORT")
|
||||
.unwrap_or_else(|_| "8333".to_owned())
|
||||
.parse()?,
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
wait_http(
|
||||
"SeaweedFS master",
|
||||
&env::var("SEAWEEDFS_MASTER_URL")
|
||||
.unwrap_or_else(|_| "http://127.0.0.1:9333/cluster/status".to_owned()),
|
||||
timeout,
|
||||
)
|
||||
.await?;
|
||||
wait_s3_api(timeout).await?;
|
||||
if !quick {
|
||||
verify_schema(None, None).await?;
|
||||
check_s3_buckets()?;
|
||||
check_config_load()?;
|
||||
}
|
||||
if public {
|
||||
check_gateway_internal_rpc().await?;
|
||||
check_public_routes(timeout).await?;
|
||||
}
|
||||
println!("Fluxer smoke checks passed.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_config_load() -> Result<()> {
|
||||
let script = concat!(
|
||||
"void import('./packages/config/src/ConfigLoader.ts')",
|
||||
".then(async (module) => { const config = await module.loadConfig(); ",
|
||||
"if (!config.integrations.voice.enabled) throw new Error('voice disabled'); ",
|
||||
"console.log('config: ok'); })",
|
||||
".catch((error) => { console.error(error); process.exit(1); });",
|
||||
);
|
||||
crate::proc::run(&["pnpm", "exec", "tsx", "-e", script])
|
||||
}
|
||||
|
||||
pub fn s3_endpoint() -> String {
|
||||
env::var("FLUXER_S3_ENDPOINT").unwrap_or_else(|_| "http://127.0.0.1:8333".to_owned())
|
||||
}
|
||||
|
||||
pub fn s3_env() -> Vec<(String, Option<String>)> {
|
||||
vec![
|
||||
(
|
||||
"AWS_ACCESS_KEY_ID".to_owned(),
|
||||
Some(env::var("FLUXER_S3_ACCESS_KEY_ID").unwrap_or_else(|_| "fluxer".to_owned())),
|
||||
),
|
||||
(
|
||||
"AWS_SECRET_ACCESS_KEY".to_owned(),
|
||||
Some(
|
||||
env::var("FLUXER_S3_SECRET_ACCESS_KEY")
|
||||
.unwrap_or_else(|_| "fluxer-secret".to_owned()),
|
||||
),
|
||||
),
|
||||
(
|
||||
"AWS_DEFAULT_REGION".to_owned(),
|
||||
Some(env::var("FLUXER_S3_REGION").unwrap_or_else(|_| "us-east-1".to_owned())),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
pub async fn wait_s3_api(timeout_secs: u64) -> Result<()> {
|
||||
let endpoint = s3_endpoint();
|
||||
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
|
||||
let mut last_output = String::new();
|
||||
while Instant::now() < deadline {
|
||||
let env = merged_env(Some(&s3_env()), true)?;
|
||||
let output = Command::new("aws")
|
||||
.args(["--endpoint-url", &endpoint, "s3api", "list-buckets"])
|
||||
.env_clear()
|
||||
.envs(env)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.output()
|
||||
.context("failed to run aws s3api list-buckets")?;
|
||||
if output.status.success() {
|
||||
println!("SeaweedFS S3 API is reachable at {endpoint}");
|
||||
return Ok(());
|
||||
}
|
||||
let mut combined = output.stdout;
|
||||
combined.extend(output.stderr);
|
||||
last_output = String::from_utf8_lossy(&combined).trim().to_owned();
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
bail!("Timed out waiting for SeaweedFS S3 API at {endpoint}: {last_output}");
|
||||
}
|
||||
|
||||
pub fn ensure_s3_buckets() -> Result<()> {
|
||||
let endpoint = s3_endpoint();
|
||||
let env = s3_env();
|
||||
for bucket in S3_BUCKETS {
|
||||
ensure_s3_bucket(&endpoint, &env, bucket)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_s3_bucket(endpoint: &str, env: &[(String, Option<String>)], bucket: &str) -> Result<()> {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
|
||||
loop {
|
||||
if s3_bucket_exists(endpoint, env, bucket)? {
|
||||
println!("S3 bucket exists: {bucket}");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let output = create_s3_bucket(endpoint, env, bucket)?;
|
||||
if output.status.success() {
|
||||
println!("Created S3 bucket: {bucket}");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let text = command_output_text(&output);
|
||||
if !s3_bucket_already_exists_output(&text) {
|
||||
let code = output.status.code().unwrap_or(-1);
|
||||
bail!(
|
||||
"Command failed with exit code {code}: aws --endpoint-url {endpoint} s3api create-bucket --bucket {bucket}"
|
||||
);
|
||||
}
|
||||
|
||||
if Instant::now() >= deadline {
|
||||
bail!(
|
||||
"Timed out waiting for S3 bucket {bucket} to become readable after create-bucket reported it already exists: {text}"
|
||||
);
|
||||
}
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
fn s3_bucket_exists(
|
||||
endpoint: &str,
|
||||
env: &[(String, Option<String>)],
|
||||
bucket: &str,
|
||||
) -> Result<bool> {
|
||||
let output = run_command(
|
||||
&[
|
||||
"aws",
|
||||
"--endpoint-url",
|
||||
endpoint,
|
||||
"s3api",
|
||||
"head-bucket",
|
||||
"--bucket",
|
||||
bucket,
|
||||
],
|
||||
RunOptions {
|
||||
env: env.to_vec(),
|
||||
check: false,
|
||||
capture: true,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)?;
|
||||
Ok(output.status.success())
|
||||
}
|
||||
|
||||
fn create_s3_bucket(
|
||||
endpoint: &str,
|
||||
env: &[(String, Option<String>)],
|
||||
bucket: &str,
|
||||
) -> Result<Output> {
|
||||
run_command(
|
||||
&[
|
||||
"aws",
|
||||
"--endpoint-url",
|
||||
endpoint,
|
||||
"s3api",
|
||||
"create-bucket",
|
||||
"--bucket",
|
||||
bucket,
|
||||
],
|
||||
RunOptions {
|
||||
env: env.to_vec(),
|
||||
check: false,
|
||||
capture: true,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn command_output_text(output: &Output) -> String {
|
||||
let mut combined = output.stdout.clone();
|
||||
combined.extend_from_slice(&output.stderr);
|
||||
String::from_utf8_lossy(&combined).trim().to_owned()
|
||||
}
|
||||
|
||||
fn s3_bucket_already_exists_output(output: &str) -> bool {
|
||||
output.contains("BucketAlreadyExists") || output.contains("BucketAlreadyOwnedByYou")
|
||||
}
|
||||
|
||||
fn check_s3_buckets() -> Result<()> {
|
||||
let endpoint = s3_endpoint();
|
||||
let env = s3_env();
|
||||
for bucket in S3_BUCKETS {
|
||||
run_command(
|
||||
&[
|
||||
"aws",
|
||||
"--endpoint-url",
|
||||
&endpoint,
|
||||
"s3api",
|
||||
"head-bucket",
|
||||
"--bucket",
|
||||
bucket,
|
||||
],
|
||||
RunOptions {
|
||||
env: env.clone(),
|
||||
capture: true,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)?;
|
||||
}
|
||||
println!("S3 buckets: ok");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_public_routes(timeout_secs: u64) -> Result<()> {
|
||||
let base_url = public_smoke_base_url();
|
||||
wait_http(
|
||||
"dev proxy api",
|
||||
&format!("{base_url}/api/_health"),
|
||||
timeout_secs,
|
||||
)
|
||||
.await?;
|
||||
wait_http(
|
||||
"dev proxy media",
|
||||
&format!("{base_url}/media/_health"),
|
||||
timeout_secs,
|
||||
)
|
||||
.await?;
|
||||
wait_http(
|
||||
"dev proxy gateway",
|
||||
&format!("{base_url}/gateway/_health"),
|
||||
timeout_secs,
|
||||
)
|
||||
.await?;
|
||||
check_gateway_websocket(&base_url, timeout_secs).await?;
|
||||
wait_http("dev proxy app", &format!("{base_url}/"), timeout_secs).await?;
|
||||
wait_http("devmail", &format!("{base_url}/devmail/"), timeout_secs).await
|
||||
}
|
||||
|
||||
async fn check_gateway_websocket(base_url: &str, timeout_secs: u64) -> Result<()> {
|
||||
let url = gateway_websocket_url(base_url)?;
|
||||
let connect_timeout = Duration::from_secs(timeout_secs.clamp(5, 30));
|
||||
let (mut socket, _) = tokio::time::timeout(connect_timeout, connect_async(url.as_str()))
|
||||
.await
|
||||
.with_context(|| format!("timed out connecting to gateway websocket at {url}"))?
|
||||
.with_context(|| format!("failed to connect to gateway websocket at {url}"))?;
|
||||
let frame = tokio::time::timeout(Duration::from_secs(10), socket.next())
|
||||
.await
|
||||
.with_context(|| format!("timed out waiting for gateway websocket hello at {url}"))?
|
||||
.ok_or_else(|| anyhow::anyhow!("gateway websocket ended before hello at {url}"))?
|
||||
.with_context(|| format!("gateway websocket receive failed at {url}"))?;
|
||||
if matches!(frame, Message::Close(_)) {
|
||||
bail!("gateway websocket closed before hello at {url}");
|
||||
}
|
||||
println!("gateway websocket is reachable at {url}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn gateway_websocket_url(base_url: &str) -> Result<String> {
|
||||
let mut url =
|
||||
Url::parse(base_url).with_context(|| format!("invalid public URL: {base_url}"))?;
|
||||
let scheme = match url.scheme() {
|
||||
"https" => "wss",
|
||||
"http" => "ws",
|
||||
scheme => bail!("public URL must use http or https for websocket smoke: {scheme}"),
|
||||
};
|
||||
url.set_scheme(scheme)
|
||||
.map_err(|_| anyhow::anyhow!("failed to set websocket scheme for {base_url}"))?;
|
||||
url.set_path("/gateway");
|
||||
url.set_query(Some("v=1&encoding=json&compress=zstd-stream&stream=1"));
|
||||
url.set_fragment(None);
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
fn public_smoke_base_url() -> String {
|
||||
if let Ok(url) = env::var("FLUXER_DEV_PUBLIC_SMOKE_URL") {
|
||||
let url = url.trim();
|
||||
if !url.is_empty() {
|
||||
return url.trim_end_matches('/').to_owned();
|
||||
}
|
||||
}
|
||||
if let Ok(url) = env::var("FLUXER_PUBLIC_URL") {
|
||||
let url = url.trim().trim_end_matches('/');
|
||||
if !url.is_empty() && !matches!(url, "http://localhost:8088" | "http://127.0.0.1:8088") {
|
||||
return url.to_owned();
|
||||
}
|
||||
}
|
||||
if let Ok(url) = crate::tunnel::resolve_cloudflare_public_url(None) {
|
||||
return url.trim_end_matches('/').to_owned();
|
||||
}
|
||||
format!("http://{LOOPBACK_HOST}:{DEV_PROXY_PORT}")
|
||||
}
|
||||
|
||||
async fn check_gateway_internal_rpc() -> Result<()> {
|
||||
let base_endpoint = env::var("FLUXER_INTERNAL_API_ENDPOINT")
|
||||
.unwrap_or_else(|_| format!("http://{LOOPBACK_HOST}:{API_PORT}"));
|
||||
let endpoint = env::var("FLUXER_GATEWAY_API_RPC_ENDPOINT")
|
||||
.unwrap_or_else(|_| format!("{}/internal/rpc", base_endpoint.trim_end_matches('/')));
|
||||
let token = env::var("FLUXER_GATEWAY_RPC_AUTH_TOKEN").unwrap_or_default();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
headers.insert("x-forwarded-for", HeaderValue::from_static(LOOPBACK_HOST));
|
||||
headers.insert("x-fluxer-rpc-auth", HeaderValue::from_str(&token)?);
|
||||
let payload: serde_json::Value = reqwest::Client::new()
|
||||
.post(endpoint)
|
||||
.headers(headers)
|
||||
.json(&json!({"type": "get_gateway_rollout_config"}))
|
||||
.send()
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
if payload.get("type").and_then(|value| value.as_str()) != Some("get_gateway_rollout_config")
|
||||
|| payload.pointer("/data/config").is_none()
|
||||
{
|
||||
bail!("Gateway internal RPC returned unexpected payload: {payload}");
|
||||
}
|
||||
println!("Gateway internal RPC: ok");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn bootstrap_schema_and_object_store() -> Result<()> {
|
||||
apply_schema(Some(config_from_env()?)).await?;
|
||||
ensure_s3_buckets()
|
||||
}
|
||||
|
||||
pub async fn http_ok(url: &str) -> bool {
|
||||
reqwest::Client::new()
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(5))
|
||||
.send()
|
||||
.await
|
||||
.map(|response| response.status().as_u16() < 500)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn s3_env_uses_fluxer_defaults() {
|
||||
let env = s3_env();
|
||||
assert!(env.iter().any(
|
||||
|(key, value)| key == "AWS_DEFAULT_REGION" && value.as_deref() == Some("us-east-1")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_already_exists_output_matches_aws_errors() {
|
||||
assert!(s3_bucket_already_exists_output(
|
||||
"An error occurred (BucketAlreadyExists) when calling the CreateBucket operation"
|
||||
));
|
||||
assert!(s3_bucket_already_exists_output(
|
||||
"An error occurred (BucketAlreadyOwnedByYou) when calling the CreateBucket operation"
|
||||
));
|
||||
assert!(!s3_bucket_already_exists_output(
|
||||
"An error occurred (AccessDenied) when calling the CreateBucket operation"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_websocket_url_uses_public_scheme() {
|
||||
assert_eq!(
|
||||
gateway_websocket_url("https://dev.example.test").unwrap(),
|
||||
"wss://dev.example.test/gateway?v=1&encoding=json&compress=zstd-stream&stream=1"
|
||||
);
|
||||
assert_eq!(
|
||||
gateway_websocket_url("http://localhost:8088").unwrap(),
|
||||
"ws://localhost:8088/gateway?v=1&encoding=json&compress=zstd-stream&stream=1"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::desktop::build_desktop;
|
||||
use crate::proc::{RunOptions, run_command};
|
||||
use anyhow::Result;
|
||||
use std::env;
|
||||
use std::path::Path;
|
||||
|
||||
const DEFAULT_TEST_WORKSPACE_CONCURRENCY: &str = "2";
|
||||
const DEFAULT_API_TEST_WORKERS: &str = "2";
|
||||
|
||||
fn task_run(args: &[&str]) -> Result<()> {
|
||||
run_command(
|
||||
args,
|
||||
RunOptions {
|
||||
load_default_env: false,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)
|
||||
.map(drop)
|
||||
}
|
||||
|
||||
fn test_env() -> Vec<(String, Option<String>)> {
|
||||
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
|
||||
let api_workers =
|
||||
env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| DEFAULT_API_TEST_WORKERS.to_owned());
|
||||
vec![
|
||||
("FLUXER_NATS_URL".to_owned(), Some(nats_url.clone())),
|
||||
(
|
||||
"FLUXER_NATS_CORE_URL".to_owned(),
|
||||
Some(env::var("FLUXER_NATS_CORE_URL").unwrap_or_else(|_| nats_url.clone())),
|
||||
),
|
||||
(
|
||||
"FLUXER_NATS_JETSTREAM_URL".to_owned(),
|
||||
Some(env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone())),
|
||||
),
|
||||
("API_TEST_MAX_WORKERS".to_owned(), Some(api_workers.clone())),
|
||||
(
|
||||
"API_TEST_MAX_CONCURRENCY".to_owned(),
|
||||
Some(env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers)),
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
fn default_test_nats_url() -> String {
|
||||
let host = if Path::new("/.dockerenv").exists() {
|
||||
"nats"
|
||||
} else {
|
||||
"127.0.0.1"
|
||||
};
|
||||
format!("nats://{host}:4222")
|
||||
}
|
||||
|
||||
fn run_generators(for_typecheck: bool) -> Result<()> {
|
||||
task_run(&["pnpm", "--filter", "@fluxer/config", "generate"])?;
|
||||
task_run(&["pnpm", "--filter", "@fluxer/schema", "generate"])?;
|
||||
if for_typecheck {
|
||||
task_run(&["pnpm", "--filter", "@fluxer/i18n", "generate:types"])?;
|
||||
}
|
||||
task_run(&["pnpm", "--filter", "fluxer_app", "i18n:compile"])
|
||||
}
|
||||
|
||||
pub fn run_typecheck() -> Result<i32> {
|
||||
run_generators(true)?;
|
||||
task_run(&["pnpm", "-r", "--if-present", "typecheck"])?;
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
pub fn run_test() -> Result<i32> {
|
||||
run_generators(false)?;
|
||||
let workspace_concurrency = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY")
|
||||
.unwrap_or_else(|_| DEFAULT_TEST_WORKSPACE_CONCURRENCY.to_owned());
|
||||
let env = test_env();
|
||||
run_command(
|
||||
&[
|
||||
"pnpm",
|
||||
"-r",
|
||||
&format!("--workspace-concurrency={workspace_concurrency}"),
|
||||
"--filter",
|
||||
"!fluxer_api",
|
||||
"--filter",
|
||||
"!fluxer",
|
||||
"--if-present",
|
||||
"test",
|
||||
],
|
||||
RunOptions {
|
||||
env: env.clone(),
|
||||
load_default_env: false,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)?;
|
||||
run_command(
|
||||
&["pnpm", "--filter", "fluxer_api", "test"],
|
||||
RunOptions {
|
||||
env,
|
||||
load_default_env: false,
|
||||
..RunOptions::default()
|
||||
},
|
||||
)?;
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
pub fn run_build() -> Result<i32> {
|
||||
run_generators(false)?;
|
||||
task_run(&["pnpm", "--filter", "fluxer_app", "build"])?;
|
||||
build_desktop(false)?;
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
pub fn run_knip() -> Result<i32> {
|
||||
task_run(&["pnpm", "--filter", "fluxer_app", "i18n:compile"])?;
|
||||
task_run(&["pnpm", "exec", "knip"])?;
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_nats_url_switches_inside_container() {
|
||||
let expected_host = if Path::new("/.dockerenv").exists() {
|
||||
"nats"
|
||||
} else {
|
||||
"127.0.0.1"
|
||||
};
|
||||
assert_eq!(
|
||||
default_test_nats_url(),
|
||||
format!("nats://{expected_host}:4222")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL, LOOPBACK_HOST};
|
||||
use crate::paths::{DEV_STATE_DIR, ROOT, which};
|
||||
use crate::proc::wait_tcp;
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use url::Url;
|
||||
|
||||
const LOCAL_ENV_START: &str = "# BEGIN fluxer-dev public URL";
|
||||
const LOCAL_ENV_END: &str = "# END fluxer-dev public URL";
|
||||
const DEFAULT_TOKEN_FILE_NAME: &str = "cloudflare-tunnel-token";
|
||||
const DEFAULT_PUBLIC_URL_FILE_NAME: &str = "cloudflare-tunnel-public-url";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PublicUrl {
|
||||
pub base_url: String,
|
||||
pub host: String,
|
||||
pub scheme: String,
|
||||
pub public_port: u16,
|
||||
pub websocket_scheme: String,
|
||||
}
|
||||
|
||||
pub fn parse_public_url(raw: &str) -> Result<PublicUrl> {
|
||||
let url = Url::parse(raw).with_context(|| format!("invalid public URL: {raw}"))?;
|
||||
let scheme = url.scheme().to_owned();
|
||||
if scheme != "http" && scheme != "https" {
|
||||
bail!("Public URL must use http or https: {raw}");
|
||||
}
|
||||
let host = url
|
||||
.host_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("Public URL must include a host: {raw}"))?
|
||||
.to_owned();
|
||||
let path = url.path();
|
||||
if path != "/" && !path.is_empty() {
|
||||
bail!("Public URL must not include a path: {raw}");
|
||||
}
|
||||
if url.query().is_some() || url.fragment().is_some() {
|
||||
bail!("Public URL must not include a query string or fragment: {raw}");
|
||||
}
|
||||
let public_port = url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| anyhow::anyhow!("Public URL has no known port: {raw}"))?;
|
||||
let mut normalized = url;
|
||||
normalized.set_path("");
|
||||
normalized.set_query(None);
|
||||
normalized.set_fragment(None);
|
||||
Ok(PublicUrl {
|
||||
base_url: normalized.as_str().trim_end_matches('/').to_owned(),
|
||||
host,
|
||||
scheme: scheme.clone(),
|
||||
public_port,
|
||||
websocket_scheme: if scheme == "https" { "wss" } else { "ws" }.to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn public_url_env(public_url: &str) -> Result<Vec<(String, String)>> {
|
||||
let parsed = parse_public_url(public_url)?;
|
||||
let base = parsed.base_url.as_str();
|
||||
let gateway_base = format!("{}://{}", parsed.websocket_scheme, parsed.host);
|
||||
let localhost_origins = "http://localhost,http://localhost:8088";
|
||||
Ok(vec![
|
||||
("FLUXER_BASE_DOMAIN".to_owned(), parsed.host.clone()),
|
||||
("FLUXER_PUBLIC_SCHEME".to_owned(), parsed.scheme.clone()),
|
||||
(
|
||||
"FLUXER_PUBLIC_PORT".to_owned(),
|
||||
parsed.public_port.to_string(),
|
||||
),
|
||||
("FLUXER_STATIC_CDN_DOMAIN".to_owned(), parsed.host.clone()),
|
||||
("FLUXER_PUBLIC_URL".to_owned(), base.to_owned()),
|
||||
("FLUXER_API_ENDPOINT".to_owned(), format!("{base}/api")),
|
||||
(
|
||||
"FLUXER_API_CLIENT_ENDPOINT".to_owned(),
|
||||
format!("{base}/api"),
|
||||
),
|
||||
("FLUXER_APP_ENDPOINT".to_owned(), base.to_owned()),
|
||||
(
|
||||
"FLUXER_GATEWAY_ENDPOINT".to_owned(),
|
||||
format!("{gateway_base}/gateway"),
|
||||
),
|
||||
("FLUXER_MEDIA_ENDPOINT".to_owned(), format!("{base}/media")),
|
||||
("FLUXER_S3_PUBLIC_ENDPOINT".to_owned(), base.to_owned()),
|
||||
("FLUXER_S3_FORCE_PATH_STYLE".to_owned(), "true".to_owned()),
|
||||
("FLUXER_STATIC_CDN_ENDPOINT".to_owned(), base.to_owned()),
|
||||
("FLUXER_ADMIN_ENDPOINT".to_owned(), format!("{base}/admin")),
|
||||
(
|
||||
"FLUXER_MARKETING_ENDPOINT".to_owned(),
|
||||
format!("{base}/marketing"),
|
||||
),
|
||||
(
|
||||
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT".to_owned(),
|
||||
format!("{base}/media"),
|
||||
),
|
||||
(
|
||||
"FLUXER_LIVEKIT_URL".to_owned(),
|
||||
format!("{gateway_base}/livekit"),
|
||||
),
|
||||
(
|
||||
"FLUXER_LIVEKIT_WEBHOOK_URL".to_owned(),
|
||||
format!("{base}/api/webhooks/livekit"),
|
||||
),
|
||||
(
|
||||
"FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT".to_owned(),
|
||||
format!("{base}/media"),
|
||||
),
|
||||
(
|
||||
"FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT".to_owned(),
|
||||
format!("{base}/media"),
|
||||
),
|
||||
(
|
||||
"FLUXER_GATEWAY_STATIC_CDN_ENDPOINT".to_owned(),
|
||||
base.to_owned(),
|
||||
),
|
||||
(
|
||||
"FLUXER_ADMIN_OAUTH_REDIRECT_URI".to_owned(),
|
||||
format!("{base}/admin/oauth2_callback"),
|
||||
),
|
||||
("FLUXER_PASSKEY_RP_ID".to_owned(), parsed.host.clone()),
|
||||
(
|
||||
"FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS".to_owned(),
|
||||
format!("{localhost_origins},{base}"),
|
||||
),
|
||||
(
|
||||
"PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT".to_owned(),
|
||||
format!("{base}/api"),
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
pub fn apply_public_url_env(public_url: &str) -> Result<()> {
|
||||
for (key, value) in public_url_env(public_url)? {
|
||||
unsafe {
|
||||
std::env::set_var(key, value);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn public_url_env_text(public_url: &str) -> Result<String> {
|
||||
let mut text = String::new();
|
||||
text.push_str(LOCAL_ENV_START);
|
||||
text.push('\n');
|
||||
text.push_str("# Generated by `fluxer-dev tunnel configure`; safe to delete.\n");
|
||||
for (key, value) in public_url_env(public_url)? {
|
||||
text.push_str(&key);
|
||||
text.push('=');
|
||||
text.push_str(&value);
|
||||
text.push('\n');
|
||||
}
|
||||
text.push_str(LOCAL_ENV_END);
|
||||
text.push('\n');
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
pub fn write_public_url_local_env(path: &Path, public_url: &str) -> Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("failed to create {}", parent.display()))?;
|
||||
}
|
||||
let generated = public_url_env_text(public_url)?;
|
||||
let existing = fs::read_to_string(path).unwrap_or_default();
|
||||
let next = replace_marked_block(&existing, &generated);
|
||||
fs::write(path, next).with_context(|| format!("failed to write {}", path.display()))?;
|
||||
println!("Wrote public URL overrides to {}", path.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn default_token_file() -> PathBuf {
|
||||
DEV_STATE_DIR.join(DEFAULT_TOKEN_FILE_NAME)
|
||||
}
|
||||
|
||||
pub fn default_public_url_file() -> PathBuf {
|
||||
DEV_STATE_DIR.join(DEFAULT_PUBLIC_URL_FILE_NAME)
|
||||
}
|
||||
|
||||
pub fn write_cloudflare_public_url_file(public_url: &str) -> Result<PathBuf> {
|
||||
fs::create_dir_all(DEV_STATE_DIR.as_path())
|
||||
.with_context(|| format!("failed to create {}", DEV_STATE_DIR.display()))?;
|
||||
let parsed = parse_public_url(public_url)?;
|
||||
let path = default_public_url_file();
|
||||
fs::write(&path, format!("{}\n", parsed.base_url))
|
||||
.with_context(|| format!("failed to write {}", path.display()))?;
|
||||
println!("Saved Cloudflare tunnel public URL to {}", path.display());
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub fn write_cloudflare_token_file(token: &str) -> Result<PathBuf> {
|
||||
fs::create_dir_all(DEV_STATE_DIR.as_path())
|
||||
.with_context(|| format!("failed to create {}", DEV_STATE_DIR.display()))?;
|
||||
let path = default_token_file();
|
||||
fs::write(&path, format!("{}\n", token.trim()))
|
||||
.with_context(|| format!("failed to write {}", path.display()))?;
|
||||
set_private_file_mode(&path)?;
|
||||
println!("Saved Cloudflare tunnel token to {}", path.display());
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub fn resolve_cloudflare_public_url(public_url_arg: Option<&str>) -> Result<String> {
|
||||
if let Some(public_url) = public_url_arg {
|
||||
return Ok(parse_public_url(public_url)?.base_url);
|
||||
}
|
||||
for key in [
|
||||
"FLUXER_CLOUDFLARE_TUNNEL_PUBLIC_URL",
|
||||
"CLOUDFLARE_TUNNEL_PUBLIC_URL",
|
||||
] {
|
||||
if let Ok(public_url) = std::env::var(key) {
|
||||
let public_url = public_url.trim();
|
||||
if !public_url.is_empty() {
|
||||
return Ok(parse_public_url(public_url)?.base_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(public_url) = std::env::var("FLUXER_PUBLIC_URL") {
|
||||
let public_url = public_url.trim();
|
||||
if !public_url.is_empty() && public_url != LOCAL_APP_URL {
|
||||
return Ok(parse_public_url(public_url)?.base_url);
|
||||
}
|
||||
}
|
||||
let path = default_public_url_file();
|
||||
if let Ok(public_url) = fs::read_to_string(&path) {
|
||||
let public_url = public_url.trim();
|
||||
if !public_url.is_empty() {
|
||||
return Ok(parse_public_url(public_url)?.base_url);
|
||||
}
|
||||
}
|
||||
bail!(
|
||||
"Missing Cloudflare tunnel public URL. Run `pnpm dev:tunnel:configure -- --public-url https://...` or pass `pnpm dev -- --cloudflare-tunnel --public-url https://...`."
|
||||
);
|
||||
}
|
||||
|
||||
pub fn apply_cloudflare_public_url_env(public_url_arg: Option<&str>) -> Result<String> {
|
||||
let public_url = resolve_cloudflare_public_url(public_url_arg)?;
|
||||
apply_public_url_env(&public_url)?;
|
||||
Ok(public_url)
|
||||
}
|
||||
|
||||
pub async fn run_cloudflare_tunnel(
|
||||
token_arg: Option<String>,
|
||||
token_file: Option<PathBuf>,
|
||||
) -> Result<i32> {
|
||||
let token = resolve_cloudflare_token(token_arg, token_file.as_deref())?;
|
||||
wait_cloudflare_tunnel_origin().await?;
|
||||
let public_url = display_public_url();
|
||||
println!("Starting Cloudflare Tunnel for {public_url} -> http://127.0.0.1:8088");
|
||||
if let Some(binary) = resolve_cloudflared_binary() {
|
||||
let status = Command::new(binary)
|
||||
.args(["tunnel", "run", "--token", token.as_str()])
|
||||
.status()
|
||||
.context("failed to start cloudflared")?;
|
||||
return Ok(status.code().unwrap_or(1));
|
||||
}
|
||||
if running_inside_devcontainer() {
|
||||
let docker = docker_command();
|
||||
let mut command = Command::new(&docker[0]);
|
||||
command.args(&docker[1..]);
|
||||
let status = command
|
||||
.args([
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
&format!(
|
||||
"container:{}",
|
||||
std::env::var("HOSTNAME")
|
||||
.unwrap_or_else(|_| "fluxer-dev-workspace-1".to_owned())
|
||||
),
|
||||
"cloudflare/cloudflared:latest",
|
||||
"tunnel",
|
||||
"run",
|
||||
"--token",
|
||||
token.as_str(),
|
||||
])
|
||||
.status()
|
||||
.context("failed to start cloudflare/cloudflared Docker image")?;
|
||||
return Ok(status.code().unwrap_or(1));
|
||||
}
|
||||
bail!(
|
||||
"cloudflared is not installed. Install cloudflared or run from the devcontainer with Docker available."
|
||||
);
|
||||
}
|
||||
|
||||
async fn wait_cloudflare_tunnel_origin() -> Result<()> {
|
||||
let timeout = std::env::var("FLUXER_CLOUDFLARE_TUNNEL_ORIGIN_READY_TIMEOUT")
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(30);
|
||||
if let Err(error) = wait_tcp("Fluxer dev proxy", LOOPBACK_HOST, DEV_PROXY_PORT, timeout).await {
|
||||
bail!(
|
||||
"Cloudflare Tunnel origin is not reachable at http://{LOOPBACK_HOST}:{DEV_PROXY_PORT}: {error}\nStart the full stack with `pnpm dev:tunnel`; `pnpm dev:tunnel:run` only starts cloudflared and expects the dev proxy to already be running."
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resolve_cloudflare_token(
|
||||
token_arg: Option<String>,
|
||||
token_file: Option<&Path>,
|
||||
) -> Result<String> {
|
||||
if let Some(token) = token_arg {
|
||||
let token = token.trim();
|
||||
if !token.is_empty() {
|
||||
return Ok(token.to_owned());
|
||||
}
|
||||
}
|
||||
for key in [
|
||||
"FLUXER_CLOUDFLARE_TUNNEL_TOKEN",
|
||||
"CLOUDFLARE_TUNNEL_TOKEN",
|
||||
"TUNNEL_TOKEN",
|
||||
] {
|
||||
if let Ok(token) = std::env::var(key) {
|
||||
let token = token.trim();
|
||||
if !token.is_empty() {
|
||||
return Ok(token.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
let path = token_file
|
||||
.map(Path::to_path_buf)
|
||||
.unwrap_or_else(default_token_file);
|
||||
if let Ok(token) = fs::read_to_string(&path) {
|
||||
let token = token.trim();
|
||||
if !token.is_empty() {
|
||||
return Ok(token.to_owned());
|
||||
}
|
||||
}
|
||||
bail!(
|
||||
"Missing Cloudflare tunnel token. Set FLUXER_CLOUDFLARE_TUNNEL_TOKEN or write {}.",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
|
||||
fn display_public_url() -> String {
|
||||
if let Ok(public_url) = std::env::var("FLUXER_PUBLIC_URL") {
|
||||
let public_url = public_url.trim();
|
||||
if !public_url.is_empty() && public_url != LOCAL_APP_URL {
|
||||
return public_url.to_owned();
|
||||
}
|
||||
}
|
||||
if let Ok(public_url) = fs::read_to_string(default_public_url_file()) {
|
||||
let public_url = public_url.trim();
|
||||
if !public_url.is_empty() {
|
||||
return public_url.to_owned();
|
||||
}
|
||||
}
|
||||
LOCAL_APP_URL.to_owned()
|
||||
}
|
||||
|
||||
fn resolve_cloudflared_binary() -> Option<PathBuf> {
|
||||
if let Ok(binary) = std::env::var("FLUXER_CLOUDFLARED_BIN")
|
||||
&& !binary.trim().is_empty()
|
||||
{
|
||||
return Some(PathBuf::from(binary));
|
||||
}
|
||||
which("cloudflared")
|
||||
}
|
||||
|
||||
fn running_inside_devcontainer() -> bool {
|
||||
ROOT.starts_with("/workspaces")
|
||||
&& Path::new("/var/run/docker.sock").exists()
|
||||
&& which("docker").is_some()
|
||||
}
|
||||
|
||||
fn docker_command() -> Vec<PathBuf> {
|
||||
if docker_socket_is_writable() || which("sudo").is_none() {
|
||||
return vec![PathBuf::from("docker")];
|
||||
}
|
||||
vec![PathBuf::from("sudo"), PathBuf::from("docker")]
|
||||
}
|
||||
|
||||
fn docker_socket_is_writable() -> bool {
|
||||
std::os::unix::net::UnixStream::connect("/var/run/docker.sock").is_ok()
|
||||
}
|
||||
|
||||
fn replace_marked_block(existing: &str, generated: &str) -> String {
|
||||
let Some(start) = existing.find(LOCAL_ENV_START) else {
|
||||
return append_block(existing, generated);
|
||||
};
|
||||
let Some(relative_end) = existing[start..].find(LOCAL_ENV_END) else {
|
||||
return append_block(existing, generated);
|
||||
};
|
||||
let end = start + relative_end + LOCAL_ENV_END.len();
|
||||
let mut next = String::new();
|
||||
next.push_str(existing[..start].trim_end());
|
||||
if !next.is_empty() {
|
||||
next.push_str("\n\n");
|
||||
}
|
||||
next.push_str(generated.trim_end());
|
||||
let suffix = existing[end..].trim_start_matches(['\r', '\n']);
|
||||
if !suffix.is_empty() {
|
||||
next.push_str("\n\n");
|
||||
next.push_str(suffix);
|
||||
} else {
|
||||
next.push('\n');
|
||||
}
|
||||
next
|
||||
}
|
||||
|
||||
fn append_block(existing: &str, generated: &str) -> String {
|
||||
let mut next = existing.trim_end().to_owned();
|
||||
if !next.is_empty() {
|
||||
next.push_str("\n\n");
|
||||
}
|
||||
next.push_str(generated.trim_end());
|
||||
next.push('\n');
|
||||
next
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn set_private_file_mode(path: &Path) -> Result<()> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut permissions = fs::metadata(path)?.permissions();
|
||||
permissions.set_mode(0o600);
|
||||
fs::set_permissions(path, permissions)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn set_private_file_mode(_path: &Path) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn derives_https_public_url_env() {
|
||||
let env = public_url_env("https://dev.example.test").unwrap();
|
||||
assert!(env.iter().any(|(key, value)| {
|
||||
key == "FLUXER_GATEWAY_ENDPOINT" && value == "wss://dev.example.test/gateway"
|
||||
}));
|
||||
assert!(env.iter().any(|(key, value)| {
|
||||
key == "FLUXER_LIVEKIT_URL" && value == "wss://dev.example.test/livekit"
|
||||
}));
|
||||
assert!(env.iter().any(|(key, value)| {
|
||||
key == "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT"
|
||||
&& value == "https://dev.example.test/media"
|
||||
}));
|
||||
assert!(env.iter().any(|(key, value)| {
|
||||
key == "FLUXER_S3_PUBLIC_ENDPOINT" && value == "https://dev.example.test"
|
||||
}));
|
||||
assert!(
|
||||
env.iter()
|
||||
.any(|(key, value)| { key == "FLUXER_PUBLIC_PORT" && value == "443" })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_public_url_with_path() {
|
||||
assert!(parse_public_url("https://example.com/app").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_public_url_arg_normalized() {
|
||||
assert_eq!(
|
||||
resolve_cloudflare_public_url(Some("https://example.com/")).unwrap(),
|
||||
"https://example.com"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaces_existing_generated_block() {
|
||||
let existing =
|
||||
"A=1\n\n# BEGIN fluxer-dev public URL\nOLD=1\n# END fluxer-dev public URL\n\nB=2\n";
|
||||
let next = replace_marked_block(existing, "NEW=1\n");
|
||||
assert_eq!(next, "A=1\n\nNEW=1\n\nB=2\n");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "fluxer-i18n-auto"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.102"
|
||||
clap = { version = "4.6.1", features = ["derive"] }
|
||||
regex = "1.12"
|
||||
reqwest = { version = "0.13.4", default-features = false, features = ["blocking", "json", "rustls"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.27.0"
|
||||
@@ -0,0 +1,212 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::config::{
|
||||
DEFAULT_OPENROUTER_APP_TITLE, DEFAULT_OPENROUTER_BASE_URL, DEFAULT_OPENROUTER_FALLBACK_MODELS,
|
||||
DEFAULT_OPENROUTER_HTTP_REFERER, DEFAULT_OPENROUTER_MODEL, DEFAULT_OPENROUTER_PROVIDER_SORT,
|
||||
EnvOverlay, env_value, trim_trailing_slash,
|
||||
};
|
||||
use crate::openrouter::is_openrouter_available;
|
||||
use crate::runner::translate_main;
|
||||
|
||||
const ENV_KEYS: &[&str] = &[
|
||||
"FLUXER_AUTO_I18N",
|
||||
"FLUXER_AUTO_I18N_LOCALE_CONCURRENCY",
|
||||
"FLUXER_AUTO_I18N_PROGRESS_INTERVAL",
|
||||
"FLUXER_AUTO_I18N_REQUEST_TIMEOUT",
|
||||
"FLUXER_AUTO_I18N_STRING_CONCURRENCY",
|
||||
"I18N_LLM_MODEL",
|
||||
"OPENROUTER_API_KEY",
|
||||
"OPENROUTER_APP_TITLE",
|
||||
"OPENROUTER_BASE_URL",
|
||||
"OPENROUTER_FALLBACK_MODELS",
|
||||
"OPENROUTER_HTTP_REFERER",
|
||||
"OPENROUTER_MODEL",
|
||||
"OPENROUTER_PROVIDER_SORT",
|
||||
];
|
||||
|
||||
pub fn auto_main(args: &[String]) -> Result<u8> {
|
||||
let env_overrides = load_env_from_files(ENV_KEYS);
|
||||
let fluxer_auto_i18n = env_value("FLUXER_AUTO_I18N", &env_overrides, "");
|
||||
let openrouter_base_url = trim_trailing_slash(&env_value(
|
||||
"OPENROUTER_BASE_URL",
|
||||
&env_overrides,
|
||||
DEFAULT_OPENROUTER_BASE_URL,
|
||||
));
|
||||
let openrouter_api_key = env_value("OPENROUTER_API_KEY", &env_overrides, "");
|
||||
let i18n_llm_model = env_value("I18N_LLM_MODEL", &env_overrides, "")
|
||||
.if_empty(|| env_value("OPENROUTER_MODEL", &env_overrides, DEFAULT_OPENROUTER_MODEL));
|
||||
let bypass_run_gate = args.iter().any(|arg| {
|
||||
matches!(arg.as_str(), "--self-test" | "--help" | "-h" | "--dry-run")
|
||||
|| arg.starts_with("--dry-run=")
|
||||
});
|
||||
let explicitly_disabled = matches!(
|
||||
fluxer_auto_i18n.to_lowercase().as_str(),
|
||||
"0" | "false" | "no" | "off"
|
||||
);
|
||||
if explicitly_disabled && !bypass_run_gate {
|
||||
eprintln!("i18n:auto skipped: FLUXER_AUTO_I18N=0 disables automatic translations.");
|
||||
return Ok(0);
|
||||
}
|
||||
let openrouter_is_available =
|
||||
is_openrouter_available(&openrouter_base_url, &openrouter_api_key);
|
||||
let should_run = fluxer_auto_i18n == "1" || (!explicitly_disabled && openrouter_is_available);
|
||||
if !should_run && !bypass_run_gate {
|
||||
eprintln!(
|
||||
"i18n:auto skipped: OpenRouter is unavailable. Set OPENROUTER_API_KEY, or set FLUXER_AUTO_I18N=1 to attempt translations with {} at {}.",
|
||||
if i18n_llm_model.is_empty() {
|
||||
DEFAULT_OPENROUTER_MODEL
|
||||
} else {
|
||||
&i18n_llm_model
|
||||
},
|
||||
openrouter_base_url
|
||||
);
|
||||
return Ok(0);
|
||||
}
|
||||
if should_run && fluxer_auto_i18n != "1" && openrouter_is_available {
|
||||
println!(
|
||||
"i18n:auto detected OpenRouter availability at {openrouter_base_url}; running without FLUXER_AUTO_I18N=1."
|
||||
);
|
||||
}
|
||||
|
||||
let mut runner_env = env_overrides;
|
||||
runner_env.insert(
|
||||
"FLUXER_AUTO_I18N".to_string(),
|
||||
if should_run {
|
||||
"1".to_string()
|
||||
} else {
|
||||
fluxer_auto_i18n
|
||||
},
|
||||
);
|
||||
runner_env.insert("I18N_LLM_MODEL".to_string(), i18n_llm_model.clone());
|
||||
runner_env.insert(
|
||||
"OPENROUTER_MODEL".to_string(),
|
||||
env_value("OPENROUTER_MODEL", &runner_env, DEFAULT_OPENROUTER_MODEL),
|
||||
);
|
||||
runner_env.insert("OPENROUTER_BASE_URL".to_string(), openrouter_base_url);
|
||||
runner_env.insert("OPENROUTER_API_KEY".to_string(), openrouter_api_key);
|
||||
runner_env.insert(
|
||||
"OPENROUTER_FALLBACK_MODELS".to_string(),
|
||||
env_value(
|
||||
"OPENROUTER_FALLBACK_MODELS",
|
||||
&runner_env,
|
||||
DEFAULT_OPENROUTER_FALLBACK_MODELS,
|
||||
),
|
||||
);
|
||||
runner_env.insert(
|
||||
"OPENROUTER_PROVIDER_SORT".to_string(),
|
||||
env_value(
|
||||
"OPENROUTER_PROVIDER_SORT",
|
||||
&runner_env,
|
||||
DEFAULT_OPENROUTER_PROVIDER_SORT,
|
||||
),
|
||||
);
|
||||
runner_env.insert(
|
||||
"OPENROUTER_HTTP_REFERER".to_string(),
|
||||
env_value(
|
||||
"OPENROUTER_HTTP_REFERER",
|
||||
&runner_env,
|
||||
DEFAULT_OPENROUTER_HTTP_REFERER,
|
||||
),
|
||||
);
|
||||
runner_env.insert(
|
||||
"OPENROUTER_APP_TITLE".to_string(),
|
||||
env_value(
|
||||
"OPENROUTER_APP_TITLE",
|
||||
&runner_env,
|
||||
DEFAULT_OPENROUTER_APP_TITLE,
|
||||
),
|
||||
);
|
||||
translate_main(args, &runner_env)
|
||||
}
|
||||
|
||||
pub fn load_env_from_files(keys: &[&str]) -> EnvOverlay {
|
||||
let target_keys = keys
|
||||
.iter()
|
||||
.copied()
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
let mut env = EnvOverlay::new();
|
||||
let Some(home) = std::env::var_os("HOME").map(PathBuf::from) else {
|
||||
return env;
|
||||
};
|
||||
for candidate in [
|
||||
".bash_profile",
|
||||
".bashrc",
|
||||
".profile",
|
||||
".zprofile",
|
||||
".zshrc",
|
||||
] {
|
||||
let file_path = home.join(candidate);
|
||||
let Ok(content) = fs::read_to_string(file_path) else {
|
||||
continue;
|
||||
};
|
||||
for line in content.lines() {
|
||||
let Some((key, value)) = parse_export_line(line) else {
|
||||
continue;
|
||||
};
|
||||
if target_keys.contains(key.as_str()) {
|
||||
env.entry(key).or_insert(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
env
|
||||
}
|
||||
|
||||
pub fn parse_export_line(line: &str) -> Option<(String, String)> {
|
||||
let trimmed = line.trim();
|
||||
let key_value = trimmed.strip_prefix("export ")?;
|
||||
let (key, value) = key_value.split_once('=')?;
|
||||
let mut chars = key.chars();
|
||||
let first = chars.next()?;
|
||||
if !(first.is_alphabetic() || first == '_') {
|
||||
return None;
|
||||
}
|
||||
if !chars.all(|character| character.is_alphanumeric() || character == '_') {
|
||||
return None;
|
||||
}
|
||||
Some((key.to_string(), strip_quotes(value)))
|
||||
}
|
||||
|
||||
pub fn strip_quotes(value: &str) -> String {
|
||||
let trimmed = value.trim();
|
||||
let first = trimmed.chars().next();
|
||||
let last = trimmed.chars().next_back();
|
||||
if trimmed.len() >= 2 && first == last && matches!(first, Some('\'' | '"')) {
|
||||
trimmed[1..trimmed.len() - 1].to_string()
|
||||
} else {
|
||||
trimmed.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
trait EmptyFallback {
|
||||
fn if_empty(self, fallback: impl FnOnce() -> String) -> String;
|
||||
}
|
||||
|
||||
impl EmptyFallback for String {
|
||||
fn if_empty(self, fallback: impl FnOnce() -> String) -> String {
|
||||
if self.is_empty() { fallback() } else { self }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_simple_export_lines() {
|
||||
assert_eq!(
|
||||
parse_export_line(" export OPENROUTER_MODEL=\"translator\" "),
|
||||
Some(("OPENROUTER_MODEL".to_string(), "translator".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
parse_export_line("export FLUXER_AUTO_I18N=1"),
|
||||
Some(("FLUXER_AUTO_I18N".to_string(), "1".to_string()))
|
||||
);
|
||||
assert_eq!(parse_export_line("OPENROUTER_MODEL=translator"), None);
|
||||
assert_eq!(parse_export_line("export 1BAD=value"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub const SOURCE_LOCALE: &str = "en-US";
|
||||
pub const DEFAULT_OPENROUTER_BASE_URL: &str = "https://openrouter.ai/api/v1";
|
||||
pub const DEFAULT_OPENROUTER_MODEL: &str = "google/gemini-2.5-flash-lite";
|
||||
pub const DEFAULT_OPENROUTER_FALLBACK_MODELS: &str =
|
||||
"openai/gpt-4.1-nano,mistralai/mistral-nemo,mistralai/mistral-small-3.2-24b-instruct";
|
||||
pub const DEFAULT_OPENROUTER_PROVIDER_SORT: &str = "throughput";
|
||||
pub const DEFAULT_OPENROUTER_HTTP_REFERER: &str = "https://fluxer.chat";
|
||||
pub const DEFAULT_OPENROUTER_APP_TITLE: &str = "Fluxer i18n auto";
|
||||
pub const DEFAULT_STRING_CONCURRENCY: usize = 2;
|
||||
pub const DEFAULT_LOCALE_CONCURRENCY: usize = 1;
|
||||
pub const DEFAULT_REQUEST_TIMEOUT_SECONDS: f64 = 300.0;
|
||||
pub const DEFAULT_PROGRESS_INTERVAL_SECONDS: f64 = 20.0;
|
||||
pub const GUIDANCE_EXCERPT_CHAR_LIMIT: usize = 1600;
|
||||
|
||||
pub const AUTO_I18N_UNCHANGED_COMMENT: &str = "# auto-i18n: reviewed unchanged";
|
||||
pub const AUTO_I18N_LEGACY_UNCHANGED_COMMENT: &str = "#. auto-i18n: reviewed unchanged";
|
||||
pub const AUTO_I18N_COMMENT_PREFIX: &str = "auto-i18n:";
|
||||
pub const AUTO_I18N_REVIEWED_UNCHANGED_FILE: &str = "auto-i18n-reviewed-unchanged.json";
|
||||
|
||||
pub fn is_auto_i18n_unchanged_comment(comment: &str) -> bool {
|
||||
let text = comment
|
||||
.trim()
|
||||
.strip_prefix("#. ")
|
||||
.or_else(|| comment.trim().strip_prefix("# "))
|
||||
.unwrap_or_else(|| comment.trim())
|
||||
.trim();
|
||||
text == "auto-i18n: reviewed unchanged"
|
||||
}
|
||||
|
||||
pub type EnvOverlay = HashMap<String, String>;
|
||||
|
||||
pub fn default_app_dir() -> PathBuf {
|
||||
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
manifest_dir
|
||||
.parent()
|
||||
.and_then(Path::parent)
|
||||
.map(|repo_root| repo_root.join("fluxer_app"))
|
||||
.unwrap_or_else(|| PathBuf::from("fluxer_app"))
|
||||
}
|
||||
|
||||
pub fn i18n_dir(app_dir: &Path) -> PathBuf {
|
||||
app_dir.join("src").join("features").join("i18n")
|
||||
}
|
||||
|
||||
pub fn locales_dir(app_dir: &Path) -> PathBuf {
|
||||
i18n_dir(app_dir).join("locales")
|
||||
}
|
||||
|
||||
pub fn env_value(key: &str, env_overrides: &EnvOverlay, fallback: &str) -> String {
|
||||
std::env::var(key)
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| env_overrides.get(key).cloned())
|
||||
.unwrap_or_else(|| fallback.to_string())
|
||||
}
|
||||
|
||||
pub fn positive_float_env(key: &str, fallback: f64, env_overrides: &EnvOverlay) -> f64 {
|
||||
let value = env_value(key, env_overrides, "");
|
||||
if value.is_empty() {
|
||||
return fallback;
|
||||
}
|
||||
value
|
||||
.parse::<f64>()
|
||||
.ok()
|
||||
.filter(|parsed| *parsed > 0.0)
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
pub fn positive_int_env(key: &str, fallback: usize, env_overrides: &EnvOverlay) -> usize {
|
||||
positive_float_env(key, fallback as f64, env_overrides).max(1.0) as usize
|
||||
}
|
||||
|
||||
pub fn trim_trailing_slash(value: &str) -> String {
|
||||
value.trim_end_matches('/').to_string()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
pub mod auto;
|
||||
pub mod config;
|
||||
pub mod llm;
|
||||
pub mod locales;
|
||||
pub mod openrouter;
|
||||
pub mod po;
|
||||
pub mod prompts;
|
||||
pub mod reviewed_unchanged;
|
||||
pub mod runner;
|
||||
pub mod tokens;
|
||||
pub mod ts_catalog;
|
||||
@@ -0,0 +1,114 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::Result;
|
||||
use regex::Regex;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct LocalizationResult {
|
||||
pub localized: String,
|
||||
pub notes: String,
|
||||
}
|
||||
|
||||
impl LocalizationResult {
|
||||
pub fn localized(localized: impl Into<String>) -> Self {
|
||||
Self {
|
||||
localized: localized.into(),
|
||||
notes: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait LocalizationClient: Sync {
|
||||
fn base_url(&self) -> &str;
|
||||
fn model(&self) -> &str;
|
||||
fn request_timeout_seconds(&self) -> f64;
|
||||
fn localize(
|
||||
&self,
|
||||
options: &Map<String, Value>,
|
||||
system_prompt: &str,
|
||||
user_prompt: &str,
|
||||
) -> Result<LocalizationResult>;
|
||||
}
|
||||
|
||||
pub fn base_options() -> Map<String, Value> {
|
||||
let mut options = Map::new();
|
||||
options.insert("temperature".to_string(), json!(0.1));
|
||||
options.insert("top_p".to_string(), json!(0.9));
|
||||
options.insert("num_ctx".to_string(), json!(4096));
|
||||
options.insert("num_predict".to_string(), json!(800));
|
||||
options
|
||||
}
|
||||
|
||||
pub fn clean_translation_response(content: &str) -> String {
|
||||
let trimmed = trim_special_response_tokens(content.trim());
|
||||
let fenced_re = Regex::new(r"(?is)^```(?:[a-z]+)?\s*([\s\S]*?)\s*```$")
|
||||
.expect("valid fenced response regex");
|
||||
let mut raw = fenced_re
|
||||
.captures(trimmed)
|
||||
.and_then(|captures| captures.get(1))
|
||||
.map(|matched| matched.as_str().trim().to_string())
|
||||
.unwrap_or_else(|| trimmed.to_string());
|
||||
let label_re = Regex::new(r"(?is)^(?:localized(?: string)?|translation|answer):\s*([\s\S]+)$")
|
||||
.expect("valid label response regex");
|
||||
if let Some(captures) = label_re.captures(&raw)
|
||||
&& let Some(value) = captures.get(1)
|
||||
{
|
||||
raw = value.as_str().trim().to_string();
|
||||
}
|
||||
if raw.starts_with('{')
|
||||
&& raw.ends_with('}')
|
||||
&& let Ok(Value::Object(object)) = serde_json::from_str::<Value>(&raw)
|
||||
&& let Some(localized) = object.get("localized").and_then(Value::as_str)
|
||||
{
|
||||
return localized.to_string();
|
||||
}
|
||||
if raw.len() >= 2
|
||||
&& raw.starts_with('"')
|
||||
&& raw.ends_with('"')
|
||||
&& let Ok(Value::String(value)) = serde_json::from_str::<Value>(&raw)
|
||||
{
|
||||
return value;
|
||||
}
|
||||
raw
|
||||
}
|
||||
|
||||
fn trim_special_response_tokens(content: &str) -> &str {
|
||||
let mut trimmed = content.trim();
|
||||
loop {
|
||||
let next = trimmed
|
||||
.strip_suffix("<|im_end|>")
|
||||
.or_else(|| trimmed.strip_suffix("<end_of_turn>"))
|
||||
.or_else(|| trimmed.strip_suffix("</s>"));
|
||||
let Some(next) = next else {
|
||||
return trimmed.trim();
|
||||
};
|
||||
trimmed = next.trim();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cleans_common_llm_response_wrappers() {
|
||||
assert_eq!(
|
||||
clean_translation_response("Translation: Bonjour"),
|
||||
"Bonjour"
|
||||
);
|
||||
assert_eq!(clean_translation_response("\"Bonjour\""), "Bonjour");
|
||||
assert_eq!(
|
||||
clean_translation_response("{\"localized\":\"Bonjour\"}"),
|
||||
"Bonjour"
|
||||
);
|
||||
assert_eq!(
|
||||
clean_translation_response("```text\nBonjour\n```"),
|
||||
"Bonjour"
|
||||
);
|
||||
assert_eq!(
|
||||
clean_translation_response("```json\n{\"0\":\"bonjour\"}\n```<|im_end|>"),
|
||||
"{\"0\":\"bonjour\"}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
pub const LOCALE_DISPLAY_NAMES: &[(&str, &str)] = &[
|
||||
("ar", "Arabic"),
|
||||
("bg", "Bulgarian"),
|
||||
("cs", "Czech"),
|
||||
("da", "Danish"),
|
||||
("de", "German"),
|
||||
("el", "Greek"),
|
||||
("en-GB", "English (United Kingdom)"),
|
||||
("es-419", "Spanish (Latin America)"),
|
||||
("es-ES", "Spanish (Spain)"),
|
||||
("fi", "Finnish"),
|
||||
("fr", "French"),
|
||||
("he", "Hebrew"),
|
||||
("hi", "Hindi"),
|
||||
("hr", "Croatian"),
|
||||
("hu", "Hungarian"),
|
||||
("id", "Indonesian"),
|
||||
("it", "Italian"),
|
||||
("ja", "Japanese"),
|
||||
("ko", "Korean"),
|
||||
("lt", "Lithuanian"),
|
||||
("nl", "Dutch"),
|
||||
("no", "Norwegian Bokmal"),
|
||||
("pl", "Polish"),
|
||||
("pt-BR", "Portuguese (Brazil)"),
|
||||
("ro", "Romanian"),
|
||||
("ru", "Russian"),
|
||||
("sv-SE", "Swedish (Sweden)"),
|
||||
("th", "Thai"),
|
||||
("tr", "Turkish"),
|
||||
("uk", "Ukrainian"),
|
||||
("vi", "Vietnamese"),
|
||||
("zh-CN", "Simplified Chinese (Mainland China)"),
|
||||
("zh-TW", "Traditional Chinese (Taiwan)"),
|
||||
];
|
||||
|
||||
pub fn display_name(locale: &str) -> &str {
|
||||
LOCALE_DISPLAY_NAMES
|
||||
.iter()
|
||||
.find_map(|(candidate, name)| (*candidate == locale).then_some(*name))
|
||||
.unwrap_or(locale)
|
||||
}
|
||||
|
||||
pub fn is_supported_locale(locale: &str) -> bool {
|
||||
LOCALE_DISPLAY_NAMES
|
||||
.iter()
|
||||
.any(|(candidate, _name)| *candidate == locale)
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::process::ExitCode;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args = std::env::args().skip(1).collect::<Vec<_>>();
|
||||
match fluxer_i18n_auto::auto::auto_main(&args) {
|
||||
Ok(code) => ExitCode::from(code),
|
||||
Err(error) => {
|
||||
eprintln!("{error:#}");
|
||||
ExitCode::from(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::llm::{LocalizationClient, LocalizationResult, clean_translation_response};
|
||||
|
||||
pub struct OpenRouterClient {
|
||||
base_url: String,
|
||||
model: String,
|
||||
fallback_models: Vec<String>,
|
||||
provider_sort: String,
|
||||
http_referer: String,
|
||||
app_title: String,
|
||||
api_key: String,
|
||||
request_timeout: Duration,
|
||||
client: reqwest::blocking::Client,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OpenRouterClientConfig {
|
||||
pub base_url: String,
|
||||
pub model: String,
|
||||
pub fallback_models: Vec<String>,
|
||||
pub provider_sort: String,
|
||||
pub http_referer: String,
|
||||
pub app_title: String,
|
||||
pub api_key: String,
|
||||
pub request_timeout_seconds: f64,
|
||||
}
|
||||
|
||||
impl OpenRouterClient {
|
||||
pub fn new(config: OpenRouterClientConfig) -> Result<Self> {
|
||||
if config.api_key.trim().is_empty() {
|
||||
bail!("OPENROUTER_API_KEY is required for i18n:auto");
|
||||
}
|
||||
let request_timeout = Duration::from_secs_f64(config.request_timeout_seconds);
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(request_timeout)
|
||||
.build()
|
||||
.context("failed to build OpenRouter HTTP client")?;
|
||||
Ok(Self {
|
||||
base_url: config.base_url.trim_end_matches('/').to_string(),
|
||||
model: config.model,
|
||||
fallback_models: config.fallback_models,
|
||||
provider_sort: config.provider_sort,
|
||||
http_referer: config.http_referer,
|
||||
app_title: config.app_title,
|
||||
api_key: config.api_key,
|
||||
request_timeout,
|
||||
client,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalizationClient for OpenRouterClient {
|
||||
fn base_url(&self) -> &str {
|
||||
&self.base_url
|
||||
}
|
||||
|
||||
fn model(&self) -> &str {
|
||||
&self.model
|
||||
}
|
||||
|
||||
fn request_timeout_seconds(&self) -> f64 {
|
||||
self.request_timeout.as_secs_f64()
|
||||
}
|
||||
|
||||
fn localize(
|
||||
&self,
|
||||
options: &Map<String, Value>,
|
||||
system_prompt: &str,
|
||||
user_prompt: &str,
|
||||
) -> Result<LocalizationResult> {
|
||||
let max_tokens = options
|
||||
.get("num_predict")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(800);
|
||||
let expects_json = system_prompt.contains("Return only valid compact JSON");
|
||||
let mut provider = Map::new();
|
||||
if !self.provider_sort.is_empty() {
|
||||
provider.insert("sort".to_string(), json!(self.provider_sort));
|
||||
}
|
||||
provider.insert("allow_fallbacks".to_string(), json!(true));
|
||||
if expects_json {
|
||||
provider.insert("require_parameters".to_string(), json!(true));
|
||||
}
|
||||
|
||||
let mut payload = Map::new();
|
||||
payload.insert("model".to_string(), json!(self.model));
|
||||
let fallback_models = self
|
||||
.fallback_models
|
||||
.iter()
|
||||
.map(|model| model.trim())
|
||||
.filter(|model| !model.is_empty() && *model != self.model)
|
||||
.map(|model| json!(model))
|
||||
.take(3)
|
||||
.collect::<Vec<_>>();
|
||||
if !fallback_models.is_empty() {
|
||||
payload.insert("models".to_string(), Value::Array(fallback_models));
|
||||
}
|
||||
payload.insert(
|
||||
"messages".to_string(),
|
||||
json!([
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]),
|
||||
);
|
||||
payload.insert(
|
||||
"temperature".to_string(),
|
||||
json!(
|
||||
options
|
||||
.get("temperature")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(0.1)
|
||||
),
|
||||
);
|
||||
payload.insert(
|
||||
"top_p".to_string(),
|
||||
json!(options.get("top_p").and_then(Value::as_f64).unwrap_or(0.9)),
|
||||
);
|
||||
payload.insert("max_tokens".to_string(), json!(max_tokens));
|
||||
payload.insert("stream".to_string(), json!(false));
|
||||
payload.insert("provider".to_string(), Value::Object(provider));
|
||||
if expects_json {
|
||||
payload.insert(
|
||||
"response_format".to_string(),
|
||||
json!({"type": "json_object"}),
|
||||
);
|
||||
}
|
||||
|
||||
let response = self
|
||||
.client
|
||||
.post(format!("{}/chat/completions", self.base_url))
|
||||
.bearer_auth(&self.api_key)
|
||||
.header("HTTP-Referer", &self.http_referer)
|
||||
.header("X-OpenRouter-Title", &self.app_title)
|
||||
.json(&Value::Object(payload))
|
||||
.send()
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"OpenRouter request failed after {}s",
|
||||
self.request_timeout.as_secs_f64()
|
||||
)
|
||||
})?;
|
||||
let status = response.status();
|
||||
let body = response
|
||||
.text()
|
||||
.context("failed to read OpenRouter response")?;
|
||||
if !status.is_success() {
|
||||
bail!(
|
||||
"OpenRouter API error: {} - {}",
|
||||
status.as_u16(),
|
||||
truncate_error_body(&body)
|
||||
);
|
||||
}
|
||||
let data = serde_json::from_str::<Value>(&body)
|
||||
.context("failed to parse OpenRouter response JSON")?;
|
||||
let Some(content) =
|
||||
extract_chat_message_content(&data).filter(|content| !content.is_empty())
|
||||
else {
|
||||
bail!("Empty response from OpenRouter");
|
||||
};
|
||||
Ok(LocalizationResult::localized(clean_translation_response(
|
||||
&content,
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_openrouter_fallback_models(value: &str) -> Vec<String> {
|
||||
value
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|model| !model.is_empty())
|
||||
.map(ToString::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn is_openrouter_available(base_url: &str, api_key: &str) -> bool {
|
||||
if api_key.trim().is_empty() {
|
||||
return false;
|
||||
}
|
||||
let Ok(client) = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(4))
|
||||
.build()
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
client
|
||||
.get(format!("{}/models", base_url.trim_end_matches('/')))
|
||||
.bearer_auth(api_key)
|
||||
.send()
|
||||
.map(|response| response.status().is_success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn extract_chat_message_content(data: &Value) -> Option<String> {
|
||||
let content = data
|
||||
.get("choices")
|
||||
.and_then(Value::as_array)
|
||||
.and_then(|choices| choices.first())
|
||||
.and_then(|choice| choice.get("message"))
|
||||
.and_then(|message| message.get("content"))?;
|
||||
match content {
|
||||
Value::String(value) => Some(value.to_string()),
|
||||
Value::Array(parts) => {
|
||||
let text = parts
|
||||
.iter()
|
||||
.filter_map(|part| match part {
|
||||
Value::String(value) => Some(value.as_str()),
|
||||
Value::Object(object) => object
|
||||
.get("text")
|
||||
.or_else(|| object.get("content"))
|
||||
.and_then(Value::as_str),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
if text.is_empty() { None } else { Some(text) }
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_error_body(body: &str) -> String {
|
||||
const MAX_ERROR_BODY_CHARS: usize = 2000;
|
||||
let mut truncated = body.chars().take(MAX_ERROR_BODY_CHARS).collect::<String>();
|
||||
if body.chars().count() > MAX_ERROR_BODY_CHARS {
|
||||
truncated.push_str("...");
|
||||
}
|
||||
truncated
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_comma_separated_fallback_models() {
|
||||
assert_eq!(
|
||||
parse_openrouter_fallback_models("a, b,,c "),
|
||||
vec!["a".to_string(), "b".to_string(), "c".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_text_from_openai_style_message() {
|
||||
let data = json!({
|
||||
"choices": [
|
||||
{"message": {"content": [{"type": "text", "text": "Hej"}]}}
|
||||
]
|
||||
});
|
||||
assert_eq!(extract_chat_message_content(&data), Some("Hej".to_string()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use regex::Regex;
|
||||
|
||||
use crate::config::{
|
||||
AUTO_I18N_COMMENT_PREFIX, AUTO_I18N_UNCHANGED_COMMENT, is_auto_i18n_unchanged_comment,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub struct Entry {
|
||||
pub comments: Vec<String>,
|
||||
pub references: Vec<String>,
|
||||
pub msgctxt: Option<String>,
|
||||
pub msgid: String,
|
||||
pub msgstr: String,
|
||||
pub line_number: usize,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
pub fn with_msgid(msgid: impl Into<String>) -> Self {
|
||||
Self {
|
||||
msgid: msgid.into(),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn with_line_number(line_number: usize) -> Self {
|
||||
Self {
|
||||
line_number,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct Translation {
|
||||
pub msgid: String,
|
||||
pub msgstr: String,
|
||||
pub msgctxt: Option<String>,
|
||||
pub reviewed_unchanged: bool,
|
||||
pub notes: String,
|
||||
}
|
||||
|
||||
impl Translation {
|
||||
pub fn new(
|
||||
msgctxt: Option<String>,
|
||||
msgid: impl Into<String>,
|
||||
msgstr: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
msgid: msgid.into(),
|
||||
msgstr: msgstr.into(),
|
||||
msgctxt,
|
||||
reviewed_unchanged: false,
|
||||
notes: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_po(content: &str) -> Result<Vec<Entry>> {
|
||||
let mut entries = Vec::new();
|
||||
let normalized = content.replace("\r\n", "\n");
|
||||
let mut current: Option<Entry> = None;
|
||||
let mut current_field: Option<Field> = None;
|
||||
let mut is_header = true;
|
||||
|
||||
for (index, line) in normalized.split('\n').enumerate() {
|
||||
if line.starts_with("#. ") || line.starts_with("# ") {
|
||||
current
|
||||
.get_or_insert_with(|| Entry::with_line_number(index))
|
||||
.comments
|
||||
.push(line.to_string());
|
||||
} else if line.starts_with("#: ") {
|
||||
current
|
||||
.get_or_insert_with(|| Entry::with_line_number(index))
|
||||
.references
|
||||
.push(line.to_string());
|
||||
} else if let Some(token) = line.strip_prefix("msgctxt ") {
|
||||
let entry = current.get_or_insert_with(|| Entry::with_line_number(index));
|
||||
entry.msgctxt = Some(parse_po_token(token, index)?);
|
||||
current_field = Some(Field::Msgctxt);
|
||||
} else if let Some(token) = line.strip_prefix("msgid ") {
|
||||
let entry = current.get_or_insert_with(|| Entry::with_line_number(index));
|
||||
entry.msgid = parse_po_token(token, index)?;
|
||||
current_field = Some(Field::Msgid);
|
||||
} else if let Some(token) = line.strip_prefix("msgstr ") {
|
||||
let entry = current.get_or_insert_with(|| Entry::with_line_number(index));
|
||||
entry.msgstr = parse_po_token(token, index)?;
|
||||
current_field = Some(Field::Msgstr);
|
||||
} else if line.starts_with('"') && line.ends_with('"') {
|
||||
if let (Some(entry), Some(field)) = (current.as_mut(), current_field) {
|
||||
let value = parse_po_token(line, index)?;
|
||||
match field {
|
||||
Field::Msgctxt => {
|
||||
if let Some(msgctxt) = entry.msgctxt.as_mut() {
|
||||
msgctxt.push_str(&value);
|
||||
}
|
||||
}
|
||||
Field::Msgid => entry.msgid.push_str(&value),
|
||||
Field::Msgstr => entry.msgstr.push_str(&value),
|
||||
}
|
||||
}
|
||||
} else if line.is_empty() && current.is_some() {
|
||||
let entry = current.take().expect("checked current exists");
|
||||
if is_header && entry.msgid.is_empty() {
|
||||
is_header = false;
|
||||
} else if !entry.msgid.is_empty() {
|
||||
entries.push(entry);
|
||||
}
|
||||
current_field = None;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(entry) = current
|
||||
&& !entry.msgid.is_empty()
|
||||
{
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Field {
|
||||
Msgctxt,
|
||||
Msgid,
|
||||
Msgstr,
|
||||
}
|
||||
|
||||
pub fn parse_po_token(token: &str, line_number: usize) -> Result<String> {
|
||||
let value = serde_json::from_str::<serde_json::Value>(token)
|
||||
.with_context(|| format!("Failed to parse PO string on line {}", line_number + 1))?;
|
||||
match value {
|
||||
serde_json::Value::String(value) => Ok(value),
|
||||
_ => bail!("PO string on line {} is not a string", line_number + 1),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn rebuild_po_allow_replacing(content: &str, translations: &[Translation]) -> Result<String> {
|
||||
let translation_map = translations
|
||||
.iter()
|
||||
.map(|translation| {
|
||||
Ok((
|
||||
entry_key(translation.msgctxt.as_deref(), &translation.msgid)?,
|
||||
translation,
|
||||
))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let normalized = content.replace("\r\n", "\n");
|
||||
let trimmed = normalized.trim_end();
|
||||
let split_re = Regex::new(r"\n{2,}").expect("valid split regex");
|
||||
let blocks = split_re.split(trimmed);
|
||||
let mut rebuilt = Vec::new();
|
||||
for block in blocks {
|
||||
rebuilt.push(rebuild_block_allow_replacing(block, &translation_map)?);
|
||||
}
|
||||
Ok(format!("{}\n", rebuilt.join("\n\n")))
|
||||
}
|
||||
|
||||
fn rebuild_block_allow_replacing(
|
||||
block: &str,
|
||||
translation_map: &[(String, &Translation)],
|
||||
) -> Result<String> {
|
||||
let lines = block
|
||||
.split('\n')
|
||||
.filter(|line| !is_auto_i18n_unchanged_comment(line))
|
||||
.map(str::to_string)
|
||||
.collect::<Vec<_>>();
|
||||
let Some(msgid_range) = field_range(&lines, "msgid") else {
|
||||
return Ok(block.to_string());
|
||||
};
|
||||
let Some(msgstr_range) = field_range(&lines, "msgstr") else {
|
||||
return Ok(block.to_string());
|
||||
};
|
||||
let msgid = read_field_value(&lines, msgid_range)?;
|
||||
let msgctxt = field_range(&lines, "msgctxt")
|
||||
.map(|range| read_field_value(&lines, range))
|
||||
.transpose()?;
|
||||
let key = entry_key(msgctxt.as_deref(), &msgid)?;
|
||||
let Some((_, translation)) = translation_map
|
||||
.iter()
|
||||
.find(|(candidate, _)| *candidate == key)
|
||||
else {
|
||||
return Ok(lines.join("\n"));
|
||||
};
|
||||
|
||||
let mut next_lines = Vec::new();
|
||||
if translation.reviewed_unchanged {
|
||||
next_lines.push(AUTO_I18N_UNCHANGED_COMMENT.to_string());
|
||||
}
|
||||
next_lines.extend_from_slice(&lines[..msgstr_range.0]);
|
||||
next_lines.push(format!("msgstr \"{}\"", escape_po(&translation.msgstr)));
|
||||
next_lines.extend_from_slice(&lines[msgstr_range.1..]);
|
||||
Ok(next_lines.join("\n"))
|
||||
}
|
||||
|
||||
pub fn reset_po_translations(content: &str) -> Result<String> {
|
||||
let translations = parse_po(content)?
|
||||
.into_iter()
|
||||
.map(|entry| Translation::new(entry.msgctxt, entry.msgid, ""))
|
||||
.collect::<Vec<_>>();
|
||||
rebuild_po_allow_replacing(content, &translations)
|
||||
}
|
||||
|
||||
pub fn entry_key(msgctxt: Option<&str>, msgid: &str) -> Result<String> {
|
||||
serde_json::to_string(&(msgctxt, msgid)).context("failed to serialize PO entry key")
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct Range(usize, usize);
|
||||
|
||||
fn field_range(lines: &[String], field_name: &str) -> Option<Range> {
|
||||
let start = lines
|
||||
.iter()
|
||||
.position(|line| line.starts_with(&format!("{field_name} ")))?;
|
||||
let mut end = start + 1;
|
||||
while end < lines.len() && lines[end].starts_with('"') && lines[end].ends_with('"') {
|
||||
end += 1;
|
||||
}
|
||||
Some(Range(start, end))
|
||||
}
|
||||
|
||||
fn read_field_value(lines: &[String], range: Range) -> Result<String> {
|
||||
let Range(start, end) = range;
|
||||
let mut value = parse_po_token(
|
||||
lines[start]
|
||||
.split_once(' ')
|
||||
.map(|(_field, token)| token)
|
||||
.unwrap_or_default(),
|
||||
start,
|
||||
)?;
|
||||
for (index, line) in lines.iter().enumerate().take(end).skip(start + 1) {
|
||||
value.push_str(&parse_po_token(line, index)?);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub fn escape_po(value: &str) -> String {
|
||||
value
|
||||
.replace('\\', "\\\\")
|
||||
.replace('"', "\\\"")
|
||||
.replace('\n', "\\n")
|
||||
.replace('\t', "\\t")
|
||||
}
|
||||
|
||||
pub fn extract_translator_comments(entry: &Entry) -> Vec<String> {
|
||||
entry
|
||||
.comments
|
||||
.iter()
|
||||
.filter_map(|line| line.strip_prefix("#. "))
|
||||
.map(str::trim)
|
||||
.filter(|comment| {
|
||||
!comment.is_empty()
|
||||
&& !is_auto_i18n_comment(comment)
|
||||
&& !is_placeholder_comment(comment)
|
||||
})
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn extract_placeholder_hints(entry: &Entry) -> Vec<String> {
|
||||
entry
|
||||
.comments
|
||||
.iter()
|
||||
.filter_map(|line| line.strip_prefix("#. "))
|
||||
.map(str::trim)
|
||||
.filter(|comment| !comment.is_empty() && is_placeholder_comment(comment))
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn is_auto_i18n_comment(comment: &str) -> bool {
|
||||
comment.to_lowercase().starts_with(AUTO_I18N_COMMENT_PREFIX)
|
||||
}
|
||||
|
||||
pub fn is_placeholder_comment(comment: &str) -> bool {
|
||||
comment.to_lowercase().starts_with("placeholder {") && comment.contains("}:")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_po() -> &'static str {
|
||||
"msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain\\n\"\n\n\
|
||||
#. Greeting shown on the welcome screen.\n\
|
||||
#. placeholder {0}: user.name\n\
|
||||
#: src/example.tsx:1\n\
|
||||
msgctxt \"welcome title\"\n\
|
||||
msgid \"Hello \\\"world\\\"\"\n\
|
||||
msgstr \"\"\n\n\
|
||||
#. auto-i18n: reviewed unchanged\n\
|
||||
#: src/example.tsx:2\n\
|
||||
msgctxt \"verb\"\n\
|
||||
msgid \"Delete\"\n\
|
||||
msgstr \"Delete\"\n\n\
|
||||
#: src/example.tsx:3\n\
|
||||
msgctxt \"keyboard key\"\n\
|
||||
msgid \"Delete\"\n\
|
||||
msgstr \"\"\n\n\
|
||||
#: src/example.tsx:4\n\
|
||||
msgid \"Line one\\n\"\n\
|
||||
\"Line two\"\n\
|
||||
msgstr \"\"\n"
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_context_comments_duplicates_and_multiline() {
|
||||
let entries = parse_po(sample_po()).unwrap();
|
||||
assert_eq!(entries.len(), 4);
|
||||
let welcome = entries
|
||||
.iter()
|
||||
.find(|entry| entry.msgctxt.as_deref() == Some("welcome title"))
|
||||
.unwrap();
|
||||
assert_eq!(welcome.msgid, "Hello \"world\"");
|
||||
assert_eq!(
|
||||
extract_translator_comments(welcome),
|
||||
vec!["Greeting shown on the welcome screen."]
|
||||
);
|
||||
assert_eq!(
|
||||
extract_placeholder_hints(welcome),
|
||||
vec!["placeholder {0}: user.name"]
|
||||
);
|
||||
let multiline = entries
|
||||
.iter()
|
||||
.find(|entry| entry.msgid.starts_with("Line"))
|
||||
.unwrap();
|
||||
assert_eq!(multiline.msgid, "Line one\nLine two");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuilds_by_context_and_manages_unchanged_markers() {
|
||||
let rebuilt = rebuild_po_allow_replacing(
|
||||
sample_po(),
|
||||
&[
|
||||
Translation::new(
|
||||
Some("welcome title".to_string()),
|
||||
"Hello \"world\"",
|
||||
"Bonjour \"monde\"",
|
||||
),
|
||||
Translation::new(Some("verb".to_string()), "Delete", "Supprimer"),
|
||||
Translation {
|
||||
msgctxt: Some("keyboard key".to_string()),
|
||||
msgid: "Delete".to_string(),
|
||||
msgstr: "Delete".to_string(),
|
||||
reviewed_unchanged: true,
|
||||
notes: String::new(),
|
||||
},
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let entries = parse_po(&rebuilt).unwrap();
|
||||
assert_eq!(
|
||||
entries
|
||||
.iter()
|
||||
.find(|entry| entry.msgctxt.as_deref() == Some("welcome title"))
|
||||
.unwrap()
|
||||
.msgstr,
|
||||
"Bonjour \"monde\""
|
||||
);
|
||||
assert_eq!(
|
||||
entries
|
||||
.iter()
|
||||
.find(|entry| entry.msgctxt.as_deref() == Some("verb"))
|
||||
.unwrap()
|
||||
.msgstr,
|
||||
"Supprimer"
|
||||
);
|
||||
let keyboard = entries
|
||||
.iter()
|
||||
.find(|entry| entry.msgctxt.as_deref() == Some("keyboard key"))
|
||||
.unwrap();
|
||||
assert_eq!(keyboard.msgstr, "Delete");
|
||||
assert!(
|
||||
keyboard
|
||||
.comments
|
||||
.contains(&AUTO_I18N_UNCHANGED_COMMENT.to_string())
|
||||
);
|
||||
assert!(
|
||||
!keyboard
|
||||
.comments
|
||||
.contains(&crate::config::AUTO_I18N_LEGACY_UNCHANGED_COMMENT.to_string())
|
||||
);
|
||||
assert!(
|
||||
!entries
|
||||
.iter()
|
||||
.find(|entry| entry.msgctxt.as_deref() == Some("verb"))
|
||||
.unwrap()
|
||||
.comments
|
||||
.contains(&AUTO_I18N_UNCHANGED_COMMENT.to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_msgstr_and_removes_reviewed_markers() {
|
||||
let reset = reset_po_translations(sample_po()).unwrap();
|
||||
let entries = parse_po(&reset).unwrap();
|
||||
assert!(entries.iter().all(|entry| entry.msgstr.is_empty()));
|
||||
assert!(!reset.contains(AUTO_I18N_UNCHANGED_COMMENT));
|
||||
assert!(!reset.contains(crate::config::AUTO_I18N_LEGACY_UNCHANGED_COMMENT));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::config::{GUIDANCE_EXCERPT_CHAR_LIMIT, locales_dir};
|
||||
use crate::locales::display_name;
|
||||
|
||||
const GLOBAL_SYSTEM_PROMPT: &str = r#"You localize product UI copy for Fluxer, a modern consumer chat app.
|
||||
|
||||
- Preserve the source intent, product function, tone, and authorial wording.
|
||||
- Translate only the source string. Context, references, and comments are metadata.
|
||||
- Preserve placeholders exactly: {name}, {{count}}, %s, %@, $1, <b>...</b>, Markdown, ICU syntax, emoji, URLs, and line breaks.
|
||||
- Preserve source sentence-fragment grammar, including leading ellipses and text that continues prior UI copy.
|
||||
- Keep copy concise and natural for product UI, without adding information that is not present in the source.
|
||||
- Keep Fluxer's product vocabulary and naming stable unless persisted locale guidance explicitly says otherwise.
|
||||
- Avoid embellishing, simplifying, softening, formalising, idiomatic rewrites, or culture-specific substitutions that change the source wording.
|
||||
- If the source is ambiguous, choose the most likely chat-app interpretation and keep the result close to the source.
|
||||
- Output only the localized string. Do not output JSON, notes, labels, quotes, or Markdown fences."#;
|
||||
|
||||
const NON_ENGLISH_SYSTEM_PROMPT: &str = r#"For non-English locales:
|
||||
|
||||
- Use target-language grammar, vocabulary, punctuation, and regional conventions.
|
||||
- Preserve the capitalisation intention of the source, adapted to target-language conventions.
|
||||
- Do not use English-style Title Case unless the target language naturally uses it.
|
||||
- Use sentence-style UI capitalisation where appropriate: usually only the first word is capitalised, and parenthesised status labels normally stay lowercase after "(" unless the word is a proper noun.
|
||||
- Prefer familiar, friendly, lightweight wording used in popular messaging apps in the target region."#;
|
||||
|
||||
const EN_GB_SYSTEM_PROMPT: &str = r#"English (United Kingdom) localisation is a minimal-edit pass over the English (United States) source.
|
||||
|
||||
- Keep Fluxer's exact wording, sentence structure, tone, and product vocabulary unless a spelling, punctuation, date, number, measurement, or grammatical locale difference requires a change.
|
||||
- Do not replace words with more British-sounding alternatives when the US wording is understandable.
|
||||
- Do not translate "US" to "UK", "United States" to "United Kingdom", or change country, region, currency, organisation, or market names unless the source text explicitly asks for that meaning change.
|
||||
- Do not add, remove, soften, formalise, idiomatically rewrite, or make the copy more akin to what a Brit would say.
|
||||
- Keep our words."#;
|
||||
|
||||
pub fn build_system_prompt(locale: &str) -> String {
|
||||
[
|
||||
Some(format!(
|
||||
"You are a professional English (United States) (en-US) to {} ({locale}) translator.",
|
||||
display_name(locale)
|
||||
)),
|
||||
Some(format!(
|
||||
"Produce exactly one {} localization for Fluxer.",
|
||||
display_name(locale)
|
||||
)),
|
||||
Some(GLOBAL_SYSTEM_PROMPT.to_string()),
|
||||
build_locale_system_prompt(locale),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n")
|
||||
}
|
||||
|
||||
pub fn build_locale_system_prompt(locale: &str) -> Option<String> {
|
||||
if locale == "en-GB" {
|
||||
Some(EN_GB_SYSTEM_PROMPT.to_string())
|
||||
} else if !locale.starts_with("en") {
|
||||
Some(NON_ENGLISH_SYSTEM_PROMPT.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_target_locale_prompt(locale: &str) -> String {
|
||||
format!(
|
||||
"Target locale: {}.\nUse the persisted locale guidance below as the source of truth for voice, tone, terminology, punctuation, and regional conventions.",
|
||||
display_name(locale)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn load_prompt_guidance(app_dir: &Path, locale: &str) -> Result<Vec<String>> {
|
||||
Ok([load_guidance_file(
|
||||
&format!("Locale guidance for {}", display_name(locale)),
|
||||
&locales_dir(app_dir)
|
||||
.join(locale)
|
||||
.join("LOCALIZATION_PROMPT.md"),
|
||||
)?]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn load_guidance_file(label: &str, file_path: &Path) -> Result<Option<String>> {
|
||||
if !file_path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let excerpt = compact_guidance_excerpt(
|
||||
&std::fs::read_to_string(file_path)?,
|
||||
GUIDANCE_EXCERPT_CHAR_LIMIT,
|
||||
);
|
||||
Ok((!excerpt.is_empty()).then(|| format!("{label}:\n{excerpt}")))
|
||||
}
|
||||
|
||||
pub fn compact_guidance_excerpt(content: &str, limit: usize) -> String {
|
||||
let mut compact = content
|
||||
.replace("\r\n", "\n")
|
||||
.split('\n')
|
||||
.map(str::trim)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
while compact.contains("\n\n\n") {
|
||||
compact = compact.replace("\n\n\n", "\n\n");
|
||||
}
|
||||
let compact = compact.trim().to_string();
|
||||
if compact.len() <= limit {
|
||||
return compact;
|
||||
}
|
||||
let limit = floor_char_boundary(&compact, limit);
|
||||
let truncated = &compact[..limit];
|
||||
let last_break = [
|
||||
truncated.rfind("\n\n"),
|
||||
truncated.rfind('\n'),
|
||||
truncated.rfind(". "),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let boundary = if last_break >= limit * 3 / 4 {
|
||||
last_break
|
||||
} else {
|
||||
limit
|
||||
};
|
||||
format!("{}\n[excerpt truncated]", compact[..boundary].trim_end())
|
||||
}
|
||||
|
||||
fn floor_char_boundary(value: &str, limit: usize) -> usize {
|
||||
let mut boundary = limit.min(value.len());
|
||||
while !value.is_char_boundary(boundary) {
|
||||
boundary -= 1;
|
||||
}
|
||||
boundary
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn en_gb_prompt_keeps_strict_preservation_guidance() {
|
||||
let prompt = build_system_prompt("en-GB");
|
||||
assert!(prompt.contains("Do not translate \"US\" to \"UK\""));
|
||||
assert!(prompt.contains("Keep our words."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_guidance_truncates_on_clean_boundary() {
|
||||
let content = "First paragraph.\n\nSecond paragraph that should be truncated.\n\nThird.";
|
||||
let compact = compact_guidance_excerpt(content, 35);
|
||||
assert!(compact.ends_with("[excerpt truncated]"));
|
||||
assert!(compact.starts_with("First paragraph."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compact_guidance_truncates_utf8_safely() {
|
||||
let content = "Brand term: caf\u{00e9} caf\u{00e9} caf\u{00e9} caf\u{00e9}.";
|
||||
let compact = compact_guidance_excerpt(content, 18);
|
||||
assert!(compact.ends_with("[excerpt truncated]"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const REVIEWED_UNCHANGED_VERSION: u32 = 1;
|
||||
const LOCK_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
|
||||
pub struct ReviewedUnchangedEntry {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub msgctxt: Option<String>,
|
||||
pub msgid: String,
|
||||
}
|
||||
|
||||
impl ReviewedUnchangedEntry {
|
||||
pub fn new(msgctxt: Option<&str>, msgid: &str) -> Self {
|
||||
Self {
|
||||
msgctxt: msgctxt.map(str::to_string),
|
||||
msgid: msgid.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
struct ReviewedUnchangedFile {
|
||||
version: u32,
|
||||
#[serde(default)]
|
||||
locales: BTreeMap<String, Vec<ReviewedUnchangedEntry>>,
|
||||
}
|
||||
|
||||
impl Default for ReviewedUnchangedFile {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: REVIEWED_UNCHANGED_VERSION,
|
||||
locales: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ReviewedUnchangedStore {
|
||||
path: PathBuf,
|
||||
data: ReviewedUnchangedFile,
|
||||
dirty_locales: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl ReviewedUnchangedStore {
|
||||
pub fn load(path: impl Into<PathBuf>) -> Result<Self> {
|
||||
let path = path.into();
|
||||
let data = read_store_file(&path)?;
|
||||
Ok(Self {
|
||||
path,
|
||||
data,
|
||||
dirty_locales: BTreeSet::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn contains(&self, locale: &str, msgctxt: Option<&str>, msgid: &str) -> bool {
|
||||
let needle = ReviewedUnchangedEntry::new(msgctxt, msgid);
|
||||
self.data
|
||||
.locales
|
||||
.get(locale)
|
||||
.is_some_and(|entries| entries.binary_search(&needle).is_ok())
|
||||
}
|
||||
|
||||
pub fn mark(&mut self, locale: &str, msgctxt: Option<&str>, msgid: &str) {
|
||||
let entry = ReviewedUnchangedEntry::new(msgctxt, msgid);
|
||||
let entries = self.data.locales.entry(locale.to_string()).or_default();
|
||||
match entries.binary_search(&entry) {
|
||||
Ok(_) => {}
|
||||
Err(index) => {
|
||||
entries.insert(index, entry);
|
||||
self.dirty_locales.insert(locale.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unmark(&mut self, locale: &str, msgctxt: Option<&str>, msgid: &str) {
|
||||
let entry = ReviewedUnchangedEntry::new(msgctxt, msgid);
|
||||
let Some(entries) = self.data.locales.get_mut(locale) else {
|
||||
return;
|
||||
};
|
||||
let Ok(index) = entries.binary_search(&entry) else {
|
||||
return;
|
||||
};
|
||||
entries.remove(index);
|
||||
if entries.is_empty() {
|
||||
self.data.locales.remove(locale);
|
||||
}
|
||||
self.dirty_locales.insert(locale.to_string());
|
||||
}
|
||||
|
||||
pub fn clear_locale(&mut self, locale: &str) {
|
||||
if self.data.locales.remove(locale).is_some() {
|
||||
self.dirty_locales.insert(locale.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn save_if_dirty(&mut self) -> Result<()> {
|
||||
if self.dirty_locales.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(parent) = self.path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("failed to create {}", parent.display()))?;
|
||||
}
|
||||
let _lock = SidecarLock::acquire(&self.path)?;
|
||||
let mut merged = read_store_file(&self.path)?;
|
||||
for locale in &self.dirty_locales {
|
||||
match self.data.locales.get(locale) {
|
||||
Some(entries) if !entries.is_empty() => {
|
||||
let mut entries = entries.clone();
|
||||
normalize_entries(&mut entries);
|
||||
merged.locales.insert(locale.clone(), entries);
|
||||
}
|
||||
_ => {
|
||||
merged.locales.remove(locale);
|
||||
}
|
||||
}
|
||||
}
|
||||
write_store_file(&self.path, &merged)?;
|
||||
self.data = merged;
|
||||
self.dirty_locales.clear();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn read_store_file(path: &Path) -> Result<ReviewedUnchangedFile> {
|
||||
let content = match fs::read_to_string(path) {
|
||||
Ok(content) => content,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => {
|
||||
return Ok(ReviewedUnchangedFile::default());
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(error).with_context(|| format!("failed to read {}", path.display()));
|
||||
}
|
||||
};
|
||||
let mut data = serde_json::from_str::<ReviewedUnchangedFile>(&content)
|
||||
.with_context(|| format!("failed to parse {}", path.display()))?;
|
||||
if data.version != REVIEWED_UNCHANGED_VERSION {
|
||||
bail!(
|
||||
"unsupported reviewed-unchanged sidecar version {} in {}",
|
||||
data.version,
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
for entries in data.locales.values_mut() {
|
||||
normalize_entries(entries);
|
||||
}
|
||||
data.locales.retain(|_, entries| !entries.is_empty());
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
fn normalize_entries(entries: &mut Vec<ReviewedUnchangedEntry>) {
|
||||
entries.sort();
|
||||
entries.dedup();
|
||||
}
|
||||
|
||||
fn write_store_file(path: &Path, data: &ReviewedUnchangedFile) -> Result<()> {
|
||||
let mut content = serde_json::to_string_pretty(data)
|
||||
.with_context(|| format!("failed to serialize {}", path.display()))?;
|
||||
content.push('\n');
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos();
|
||||
let temp_path = path.with_file_name(format!(
|
||||
".{}.{}.{}.tmp",
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("auto-i18n-reviewed-unchanged.json"),
|
||||
std::process::id(),
|
||||
timestamp
|
||||
));
|
||||
fs::write(&temp_path, content)
|
||||
.with_context(|| format!("failed to write {}", temp_path.display()))?;
|
||||
fs::rename(&temp_path, path)
|
||||
.with_context(|| format!("failed to replace {}", path.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SidecarLock {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SidecarLock {
|
||||
fn acquire(sidecar_path: &Path) -> Result<Self> {
|
||||
let lock_path = sidecar_path.with_extension("json.lock");
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
match OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&lock_path)
|
||||
{
|
||||
Ok(_) => {
|
||||
return Ok(Self { path: lock_path });
|
||||
}
|
||||
Err(error) if error.kind() == ErrorKind::AlreadyExists => {
|
||||
if started.elapsed() >= LOCK_TIMEOUT {
|
||||
bail!(
|
||||
"timed out waiting for reviewed-unchanged sidecar lock {}",
|
||||
lock_path.display()
|
||||
);
|
||||
}
|
||||
thread::sleep(LOCK_POLL_INTERVAL);
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(error)
|
||||
.with_context(|| format!("failed to create {}", lock_path.display()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SidecarLock {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn stores_entries_sorted_and_deduplicated() {
|
||||
let temp = tempdir().unwrap();
|
||||
let path = temp.path().join("reviewed.json");
|
||||
let mut store = ReviewedUnchangedStore::load(&path).unwrap();
|
||||
store.mark("de", Some("button"), "Save");
|
||||
store.mark("de", None, "Audio");
|
||||
store.mark("de", None, "Audio");
|
||||
store.save_if_dirty().unwrap();
|
||||
|
||||
let saved = fs::read_to_string(&path).unwrap();
|
||||
assert!(saved.contains("\"version\": 1"));
|
||||
|
||||
let loaded = ReviewedUnchangedStore::load(&path).unwrap();
|
||||
assert!(loaded.contains("de", None, "Audio"));
|
||||
assert!(loaded.contains("de", Some("button"), "Save"));
|
||||
assert!(!loaded.contains("de", Some("button"), "Audio"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_save_preserves_other_locale_changes() {
|
||||
let temp = tempdir().unwrap();
|
||||
let path = temp.path().join("reviewed.json");
|
||||
let mut first = ReviewedUnchangedStore::load(&path).unwrap();
|
||||
let mut second = ReviewedUnchangedStore::load(&path).unwrap();
|
||||
|
||||
first.mark("de", None, "Audio");
|
||||
second.mark("fr", None, "Avatar");
|
||||
first.save_if_dirty().unwrap();
|
||||
second.save_if_dirty().unwrap();
|
||||
|
||||
let loaded = ReviewedUnchangedStore::load(&path).unwrap();
|
||||
assert!(loaded.contains("de", None, "Audio"));
|
||||
assert!(loaded.contains("fr", None, "Avatar"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,445 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::cmp::Reverse;
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use regex::Regex;
|
||||
|
||||
use crate::po::Entry;
|
||||
|
||||
const EN_GB_PROTECTED_REGION_TERMS: &[&str] = &[
|
||||
"United States of America",
|
||||
"United States",
|
||||
"U.S.",
|
||||
"USA",
|
||||
"US",
|
||||
];
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct IcuControl {
|
||||
pub argument: String,
|
||||
pub kind: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct TokenAlias {
|
||||
pub token: String,
|
||||
pub alias: String,
|
||||
}
|
||||
|
||||
pub fn should_keep_unchanged(source: &str, locale: &str) -> bool {
|
||||
if locale == "en-US" {
|
||||
return false;
|
||||
}
|
||||
if source.trim().is_empty() {
|
||||
return true;
|
||||
}
|
||||
if !has_any_letter(source) {
|
||||
return true;
|
||||
}
|
||||
if is_preserved_token_composition(source) {
|
||||
return true;
|
||||
}
|
||||
is_preserved_artifact(source)
|
||||
}
|
||||
|
||||
pub fn has_any_letter(source: &str) -> bool {
|
||||
source.chars().any(char::is_alphabetic)
|
||||
}
|
||||
|
||||
pub fn is_preserved_artifact(source: &str) -> bool {
|
||||
let trimmed = source.trim();
|
||||
if [
|
||||
r"(?i)^https?://\S+$",
|
||||
r"(?i)^(?:mailto:|tel:|file:|app://|fluxer://)\S+$",
|
||||
r"^[@#][\w-]+$",
|
||||
r"^\{[^{}]+\}$",
|
||||
r"^<[^>]+>$",
|
||||
r"^%[@sd]$",
|
||||
r"^\$\d+$",
|
||||
r#"^[\s{}#,._:/+()\[\]'"-]+$"#,
|
||||
r"^[A-Z][A-Z0-9_./:+-]+$",
|
||||
]
|
||||
.iter()
|
||||
.any(|pattern| {
|
||||
Regex::new(pattern)
|
||||
.expect("valid artifact regex")
|
||||
.is_match(trimmed)
|
||||
}) {
|
||||
return true;
|
||||
}
|
||||
Regex::new(r"^[\w./:+-]+$")
|
||||
.expect("valid word artifact regex")
|
||||
.is_match(trimmed)
|
||||
&& Regex::new(r"[\d_./:+-]")
|
||||
.expect("valid word artifact marker regex")
|
||||
.is_match(trimmed)
|
||||
}
|
||||
|
||||
pub fn is_preserved_token_composition(source: &str) -> bool {
|
||||
let entry = Entry {
|
||||
msgid: source.to_string(),
|
||||
..Entry::default()
|
||||
};
|
||||
let mut tokens = extract_preserved_tokens(&entry);
|
||||
if tokens.is_empty() {
|
||||
return false;
|
||||
}
|
||||
tokens.sort_by_key(|token| Reverse(token.len()));
|
||||
let mut remainder = source.to_string();
|
||||
for token in tokens {
|
||||
remainder = Regex::new(®ex::escape(&token))
|
||||
.expect("valid preserved token cleanup regex")
|
||||
.replace_all(&remainder, "")
|
||||
.into_owned();
|
||||
}
|
||||
remainder
|
||||
.chars()
|
||||
.all(|character| character.is_whitespace() || is_preserved_token_separator(character))
|
||||
}
|
||||
|
||||
fn is_preserved_token_separator(character: char) -> bool {
|
||||
matches!(
|
||||
character,
|
||||
':' | ','
|
||||
| '.'
|
||||
| '/'
|
||||
| '\\'
|
||||
| '-'
|
||||
| '–'
|
||||
| '—'
|
||||
| '('
|
||||
| ')'
|
||||
| '['
|
||||
| ']'
|
||||
| '{'
|
||||
| '}'
|
||||
| '|'
|
||||
| '·'
|
||||
| '•'
|
||||
| '#'
|
||||
| '+'
|
||||
| '\''
|
||||
| '"'
|
||||
)
|
||||
}
|
||||
|
||||
pub fn validate_localization(entry: &Entry, localized: &str) -> Result<()> {
|
||||
if !entry.msgid.trim().is_empty() && localized.trim().is_empty() {
|
||||
bail!("Localized string is empty");
|
||||
}
|
||||
let source_tokens = extract_preserved_tokens(entry);
|
||||
let localized_entry = Entry {
|
||||
msgid: localized.to_string(),
|
||||
comments: entry.comments.clone(),
|
||||
..Entry::default()
|
||||
};
|
||||
let localized_tokens = extract_preserved_tokens(&localized_entry);
|
||||
for token in source_tokens {
|
||||
if !localized_tokens.contains(&token) {
|
||||
bail!("Localized string did not preserve token {token}");
|
||||
}
|
||||
}
|
||||
for control in extract_icu_controls(&entry.msgid) {
|
||||
let pattern = Regex::new(&format!(
|
||||
r"\{{\s*{}\s*,\s*{}\s*,",
|
||||
regex::escape(&control.argument),
|
||||
regex::escape(&control.kind)
|
||||
))
|
||||
.expect("valid ICU validation regex");
|
||||
if !pattern.is_match(localized) {
|
||||
bail!(
|
||||
"Localized string did not preserve ICU {} argument {}",
|
||||
control.kind,
|
||||
control.argument
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_locale_specific_localization(
|
||||
entry: &Entry,
|
||||
locale: &str,
|
||||
localized: &str,
|
||||
) -> Result<()> {
|
||||
if locale != "en-GB" {
|
||||
return Ok(());
|
||||
}
|
||||
for term in extract_en_gb_protected_terms(&entry.msgid) {
|
||||
if !contains_exact_term(localized, term) {
|
||||
bail!("en-GB localization changed protected source term {term}");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn extract_en_gb_protected_terms(source: &str) -> Vec<&'static str> {
|
||||
EN_GB_PROTECTED_REGION_TERMS
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|term| contains_exact_term(source, term))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn contains_exact_term(source: &str, term: &str) -> bool {
|
||||
source.match_indices(term).any(|(start, _match)| {
|
||||
let before_ok = source[..start]
|
||||
.chars()
|
||||
.next_back()
|
||||
.is_none_or(|before| !is_word_char(before));
|
||||
let end = start + term.len();
|
||||
let after_ok = source[end..]
|
||||
.chars()
|
||||
.next()
|
||||
.is_none_or(|after| !is_word_char(after));
|
||||
before_ok && after_ok
|
||||
})
|
||||
}
|
||||
|
||||
fn is_word_char(value: char) -> bool {
|
||||
value == '_' || value.is_alphanumeric()
|
||||
}
|
||||
|
||||
pub fn normalize_localized_capitalization(entry: &Entry, locale: &str, localized: &str) -> String {
|
||||
if locale.starts_with("en") || !entry.msgid.starts_with('(') || !localized.starts_with('(') {
|
||||
return localized.to_string();
|
||||
}
|
||||
let mut chars = localized.chars();
|
||||
let Some(first) = chars.next() else {
|
||||
return localized.to_string();
|
||||
};
|
||||
let Some(second) = chars.next() else {
|
||||
return localized.to_string();
|
||||
};
|
||||
let second_string = second.to_string();
|
||||
if second_string != second.to_uppercase().to_string() {
|
||||
return localized.to_string();
|
||||
}
|
||||
let rest = chars.as_str();
|
||||
format!("{first}{}{rest}", second.to_lowercase())
|
||||
}
|
||||
|
||||
pub fn extract_preserved_tokens(entry: &Entry) -> Vec<String> {
|
||||
let mut tokens = Vec::new();
|
||||
for token in extract_top_level_brace_tokens(&entry.msgid) {
|
||||
push_unique(&mut tokens, token);
|
||||
}
|
||||
let token_re = Regex::new(r"%[@sd]|\$\d+|<[^>]+>|https?://\S+|:[a-zA-Z0-9_+-]+:")
|
||||
.expect("valid preserved token regex");
|
||||
for matched in token_re.find_iter(&entry.msgid) {
|
||||
push_unique(&mut tokens, matched.as_str().to_string());
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
pub fn extract_top_level_brace_tokens(source: &str) -> Vec<String> {
|
||||
let mut tokens = Vec::new();
|
||||
let mut depth = 0usize;
|
||||
let mut start: Option<usize> = None;
|
||||
let double_brace_re = Regex::new(r"^\{\{[^{}]+\}\}$").expect("valid double brace regex");
|
||||
let brace_re = Regex::new(r"^\{[^{},]+\}$").expect("valid brace regex");
|
||||
for (index, character) in source.char_indices() {
|
||||
if character == '{' {
|
||||
if depth == 0 {
|
||||
start = Some(index);
|
||||
}
|
||||
depth += 1;
|
||||
} else if character == '}' && depth > 0 {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
if let Some(start_index) = start {
|
||||
let token = &source[start_index..index + character.len_utf8()];
|
||||
if double_brace_re.is_match(token) || brace_re.is_match(token) {
|
||||
tokens.push(token.to_string());
|
||||
}
|
||||
}
|
||||
start = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
tokens
|
||||
}
|
||||
|
||||
pub fn has_icu_control(source: &str) -> bool {
|
||||
Regex::new(r"\{[^{}]+,\s*(?:plural|select|selectordinal)\s*,")
|
||||
.expect("valid ICU detection regex")
|
||||
.is_match(source)
|
||||
}
|
||||
|
||||
pub fn extract_icu_controls(source: &str) -> Vec<IcuControl> {
|
||||
Regex::new(r"\{\s*([^{}\s,]+)\s*,\s*(plural|select|selectordinal)\s*,")
|
||||
.expect("valid ICU control regex")
|
||||
.captures_iter(source)
|
||||
.map(|captures| IcuControl {
|
||||
argument: captures[1].to_string(),
|
||||
kind: captures[2].to_string(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn build_masked_source(entry: &Entry) -> (String, Vec<TokenAlias>) {
|
||||
let token_aliases = extract_preserved_tokens(entry)
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, token)| TokenAlias {
|
||||
token,
|
||||
alias: format!("{{FLUXER_TOKEN_{index}}}"),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(
|
||||
mask_text_with_aliases(&entry.msgid, &token_aliases),
|
||||
token_aliases,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn mask_text_with_aliases(text: &str, token_aliases: &[TokenAlias]) -> String {
|
||||
let mut masked = text.to_string();
|
||||
let mut aliases = token_aliases.iter().collect::<Vec<_>>();
|
||||
aliases.sort_by_key(|alias| std::cmp::Reverse(alias.token.len()));
|
||||
for item in aliases {
|
||||
masked = Regex::new(®ex::escape(&item.token))
|
||||
.expect("valid token mask regex")
|
||||
.replace_all(&masked, item.alias.as_str())
|
||||
.into_owned();
|
||||
}
|
||||
masked
|
||||
}
|
||||
|
||||
pub fn restore_masked_tokens(localized: &str, token_aliases: &[TokenAlias]) -> String {
|
||||
let mut restored = localized.to_string();
|
||||
for item in token_aliases {
|
||||
restored = restored.replace(&item.alias, &item.token);
|
||||
}
|
||||
restored
|
||||
}
|
||||
|
||||
pub fn build_token_alias_context(token_aliases: &[TokenAlias]) -> String {
|
||||
let mut lines = vec!["Literal protected aliases used in the source string:".to_string()];
|
||||
lines.extend(
|
||||
token_aliases
|
||||
.iter()
|
||||
.map(|item| format!("{} = {}", item.alias, item.token)),
|
||||
);
|
||||
lines.extend([
|
||||
"Aliases are required substrings, not words or labels to localize.".to_string(),
|
||||
"Copy every alias exactly in the localized output, with the same spelling and braces.".to_string(),
|
||||
"Do not replace an alias with translated words, even if the original token name looks meaningful."
|
||||
.to_string(),
|
||||
"Translate only the human-readable words around aliases.".to_string(),
|
||||
]);
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
fn push_unique(tokens: &mut Vec<String>, token: String) {
|
||||
if !tokens.contains(&token) {
|
||||
tokens.push(token);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn detects_sources_that_should_stay_unchanged() {
|
||||
assert!(should_keep_unchanged("", "de"));
|
||||
assert!(should_keep_unchanged("https://fluxer.app/docs", "de"));
|
||||
assert!(should_keep_unchanged("{productName}", "de"));
|
||||
assert!(should_keep_unchanged("{authorName} {description}", "de"));
|
||||
assert!(should_keep_unchanged(
|
||||
"{emojiName}: {reactionCountText}, {actionText}",
|
||||
"de"
|
||||
));
|
||||
assert!(should_keep_unchanged("{start} – {end}", "de"));
|
||||
assert!(should_keep_unchanged("C++", "de"));
|
||||
assert!(!should_keep_unchanged("NCMEC {ncmecReportId}", "de"));
|
||||
assert!(!should_keep_unchanged("Hello", "de"));
|
||||
assert!(!should_keep_unchanged("{productName}", "en-US"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_masks_and_restores_preserved_tokens() {
|
||||
let entry = Entry::with_msgid("{productName} needs <0>{permission}</0> %@ $1 :wave:");
|
||||
assert_eq!(
|
||||
extract_preserved_tokens(&entry),
|
||||
vec![
|
||||
"{productName}",
|
||||
"{permission}",
|
||||
"<0>",
|
||||
"</0>",
|
||||
"%@",
|
||||
"$1",
|
||||
":wave:"
|
||||
]
|
||||
);
|
||||
let (masked, aliases) = build_masked_source(&entry);
|
||||
assert!(masked.contains("{FLUXER_TOKEN_0}"));
|
||||
assert_eq!(
|
||||
restore_masked_tokens(&masked, &aliases),
|
||||
"{productName} needs <0>{permission}</0> %@ $1 :wave:"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validates_tokens_and_icu_controls() {
|
||||
let entry = Entry {
|
||||
msgid: "{0} {1, plural, one {member} other {members}}".to_string(),
|
||||
comments: vec!["#. placeholder {0}: count".to_string()],
|
||||
..Entry::default()
|
||||
};
|
||||
validate_localization(&entry, "{0} {1, plural, one {miembro} other {miembros}}").unwrap();
|
||||
let error =
|
||||
validate_localization(&entry, "{0} {2, plural, one {x} other {x}}").unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("did not preserve ICU plural argument 1")
|
||||
);
|
||||
let missing_token =
|
||||
validate_localization(&Entry::with_msgid("Open {productName}"), "Open").unwrap_err();
|
||||
assert!(missing_token.to_string().contains("{productName}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validates_en_gb_protected_region_terms() {
|
||||
let entry = Entry::with_msgid("US color settings for the United States");
|
||||
validate_locale_specific_localization(
|
||||
&entry,
|
||||
"en-GB",
|
||||
"US colour settings for the United States",
|
||||
)
|
||||
.unwrap();
|
||||
let error = validate_locale_specific_localization(
|
||||
&entry,
|
||||
"en-GB",
|
||||
"UK colour settings for the United Kingdom",
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("protected source term United States")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalizes_non_english_parenthesized_capitalization() {
|
||||
assert_eq!(
|
||||
normalize_localized_capitalization(
|
||||
&Entry::with_msgid("(No content)"),
|
||||
"es-ES",
|
||||
"(Sin contenido)",
|
||||
),
|
||||
"(sin contenido)"
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_localized_capitalization(
|
||||
&Entry::with_msgid("(No content)"),
|
||||
"en-GB",
|
||||
"(No content)",
|
||||
),
|
||||
"(No content)"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,867 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
|
||||
use crate::po::{Entry, Translation};
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum StaticTsCatalogKind {
|
||||
SimpleMessages,
|
||||
EmailTemplates,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct StaticTsCatalogConfig {
|
||||
pub kind: StaticTsCatalogKind,
|
||||
pub source_path: PathBuf,
|
||||
pub source_export: String,
|
||||
pub locale_function: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct ObjectRange {
|
||||
body: Range<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct ParsedStringProperty {
|
||||
key: String,
|
||||
value: String,
|
||||
value_range: Range<usize>,
|
||||
line_number: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct ParsedObjectProperty {
|
||||
key: String,
|
||||
body: Range<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct StaticSourceMessage {
|
||||
context: String,
|
||||
source: String,
|
||||
line_number: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct StaticTargetCatalog {
|
||||
object_body: Range<usize>,
|
||||
values: HashMap<String, ParsedStringProperty>,
|
||||
template_objects: HashMap<String, ParsedObjectProperty>,
|
||||
}
|
||||
|
||||
pub fn read_static_ts_entries(
|
||||
config: &StaticTsCatalogConfig,
|
||||
source_content: &str,
|
||||
target_content: &str,
|
||||
reset: bool,
|
||||
) -> Result<Vec<Entry>> {
|
||||
let source_messages = parse_source_messages(config, source_content)?;
|
||||
let target_catalog = parse_target_catalog(config, target_content)?;
|
||||
Ok(source_messages
|
||||
.into_iter()
|
||||
.map(|source| {
|
||||
let msgstr = if reset {
|
||||
String::new()
|
||||
} else {
|
||||
target_catalog
|
||||
.values
|
||||
.get(&source.context)
|
||||
.map(|target| target.value.clone())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
Entry {
|
||||
comments: static_comments(config, &source.context),
|
||||
references: vec![format!(
|
||||
"#: {}:{}",
|
||||
config.source_path.display(),
|
||||
source.line_number
|
||||
)],
|
||||
msgctxt: Some(source.context),
|
||||
msgid: source.source,
|
||||
msgstr,
|
||||
line_number: source.line_number,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn rebuild_static_ts_allow_replacing(
|
||||
config: &StaticTsCatalogConfig,
|
||||
source_content: &str,
|
||||
target_content: &str,
|
||||
translations: &[Translation],
|
||||
) -> Result<String> {
|
||||
let source_messages = parse_source_messages(config, source_content)?;
|
||||
let target_catalog = parse_target_catalog(config, target_content)?;
|
||||
let translations_by_context = translations
|
||||
.iter()
|
||||
.filter_map(|translation| {
|
||||
translation
|
||||
.msgctxt
|
||||
.as_ref()
|
||||
.map(|context| (context.clone(), translation))
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut replacements = Vec::new();
|
||||
for (context, translation) in &translations_by_context {
|
||||
if let Some(target) = target_catalog.values.get(context) {
|
||||
replacements.push((
|
||||
target.value_range.clone(),
|
||||
escape_ts_single_quoted(&translation.msgstr),
|
||||
));
|
||||
}
|
||||
}
|
||||
match config.kind {
|
||||
StaticTsCatalogKind::SimpleMessages => {
|
||||
add_missing_simple_translations(
|
||||
target_content,
|
||||
&target_catalog,
|
||||
&source_messages,
|
||||
&translations_by_context,
|
||||
&mut replacements,
|
||||
);
|
||||
}
|
||||
StaticTsCatalogKind::EmailTemplates => {
|
||||
add_missing_email_translations(
|
||||
target_content,
|
||||
&target_catalog,
|
||||
&source_messages,
|
||||
&translations_by_context,
|
||||
&mut replacements,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
apply_replacements(target_content, replacements)
|
||||
}
|
||||
|
||||
pub fn reset_static_ts_translations(
|
||||
config: &StaticTsCatalogConfig,
|
||||
target_content: &str,
|
||||
) -> Result<String> {
|
||||
let target_catalog = parse_target_catalog(config, target_content)?;
|
||||
let replacements = target_catalog
|
||||
.values
|
||||
.values()
|
||||
.map(|target| (target.value_range.clone(), "''".to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
apply_replacements(target_content, replacements)
|
||||
}
|
||||
|
||||
fn static_comments(config: &StaticTsCatalogConfig, context: &str) -> Vec<String> {
|
||||
match config.kind {
|
||||
StaticTsCatalogKind::SimpleMessages => {
|
||||
vec![format!("#. Static catalog key: {context}")]
|
||||
}
|
||||
StaticTsCatalogKind::EmailTemplates => {
|
||||
vec![format!("#. Email template field: {context}")]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_source_messages(
|
||||
config: &StaticTsCatalogConfig,
|
||||
source_content: &str,
|
||||
) -> Result<Vec<StaticSourceMessage>> {
|
||||
let object = find_const_object(source_content, &config.source_export)?;
|
||||
match config.kind {
|
||||
StaticTsCatalogKind::SimpleMessages => {
|
||||
parse_string_properties(source_content, &object.body).map(|properties| {
|
||||
properties
|
||||
.into_iter()
|
||||
.map(|property| StaticSourceMessage {
|
||||
context: property.key,
|
||||
source: property.value,
|
||||
line_number: property.line_number,
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
StaticTsCatalogKind::EmailTemplates => {
|
||||
let mut messages = Vec::new();
|
||||
for template in parse_object_properties(source_content, &object.body)? {
|
||||
for field in parse_string_properties(source_content, &template.body)? {
|
||||
if field.key == "subject" || field.key == "body" {
|
||||
messages.push(StaticSourceMessage {
|
||||
context: format!("{}.{}", template.key, field.key),
|
||||
source: field.value,
|
||||
line_number: field.line_number,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(messages)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_target_catalog(
|
||||
config: &StaticTsCatalogConfig,
|
||||
target_content: &str,
|
||||
) -> Result<StaticTargetCatalog> {
|
||||
let object = find_call_object(target_content, &config.locale_function)?;
|
||||
match config.kind {
|
||||
StaticTsCatalogKind::SimpleMessages => {
|
||||
let values = parse_string_properties(target_content, &object.body)?
|
||||
.into_iter()
|
||||
.map(|property| (property.key.clone(), property))
|
||||
.collect::<HashMap<_, _>>();
|
||||
Ok(StaticTargetCatalog {
|
||||
object_body: object.body,
|
||||
values,
|
||||
template_objects: HashMap::new(),
|
||||
})
|
||||
}
|
||||
StaticTsCatalogKind::EmailTemplates => {
|
||||
let mut values = HashMap::new();
|
||||
let mut template_objects = HashMap::new();
|
||||
for template in parse_object_properties(target_content, &object.body)? {
|
||||
for field in parse_string_properties(target_content, &template.body)? {
|
||||
if field.key == "subject" || field.key == "body" {
|
||||
values.insert(format!("{}.{}", template.key, field.key), field);
|
||||
}
|
||||
}
|
||||
template_objects.insert(template.key.clone(), template);
|
||||
}
|
||||
Ok(StaticTargetCatalog {
|
||||
object_body: object.body,
|
||||
values,
|
||||
template_objects,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_missing_simple_translations(
|
||||
target_content: &str,
|
||||
target_catalog: &StaticTargetCatalog,
|
||||
source_messages: &[StaticSourceMessage],
|
||||
translations_by_context: &HashMap<String, &Translation>,
|
||||
replacements: &mut Vec<(Range<usize>, String)>,
|
||||
) {
|
||||
let lines = source_messages
|
||||
.iter()
|
||||
.filter(|source| !target_catalog.values.contains_key(&source.context))
|
||||
.filter_map(|source| {
|
||||
translations_by_context
|
||||
.get(&source.context)
|
||||
.map(|translation| {
|
||||
format!(
|
||||
"\t{}: {},",
|
||||
escape_property_key(&source.context),
|
||||
escape_ts_single_quoted(&translation.msgstr)
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if !lines.is_empty() {
|
||||
replacements.push(append_to_object_body(
|
||||
target_content,
|
||||
&target_catalog.object_body,
|
||||
lines,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
fn add_missing_email_translations(
|
||||
target_content: &str,
|
||||
target_catalog: &StaticTargetCatalog,
|
||||
source_messages: &[StaticSourceMessage],
|
||||
translations_by_context: &HashMap<String, &Translation>,
|
||||
replacements: &mut Vec<(Range<usize>, String)>,
|
||||
) -> Result<()> {
|
||||
let source_by_context = source_messages
|
||||
.iter()
|
||||
.map(|source| (source.context.clone(), source.source.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut source_template_order = Vec::new();
|
||||
for source in source_messages {
|
||||
let (template_key, _field) = split_email_context(&source.context)?;
|
||||
if !source_template_order
|
||||
.iter()
|
||||
.any(|candidate| candidate == template_key)
|
||||
{
|
||||
source_template_order.push(template_key.to_string());
|
||||
}
|
||||
}
|
||||
for template_key in &source_template_order {
|
||||
if let Some(template) = target_catalog.template_objects.get(template_key) {
|
||||
let lines = ["subject", "body"]
|
||||
.into_iter()
|
||||
.filter_map(|field| {
|
||||
let context = format!("{template_key}.{field}");
|
||||
if target_catalog.values.contains_key(&context) {
|
||||
return None;
|
||||
}
|
||||
translations_by_context.get(&context).map(|translation| {
|
||||
format!(
|
||||
"\t\t{field}: {},",
|
||||
escape_ts_single_quoted(&translation.msgstr)
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if !lines.is_empty() {
|
||||
replacements.push(append_to_object_body(target_content, &template.body, lines));
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut template_blocks = Vec::new();
|
||||
for template_key in &source_template_order {
|
||||
if target_catalog.template_objects.contains_key(template_key) {
|
||||
continue;
|
||||
}
|
||||
let subject_context = format!("{template_key}.subject");
|
||||
let body_context = format!("{template_key}.body");
|
||||
let has_selected_translation = translations_by_context.contains_key(&subject_context)
|
||||
|| translations_by_context.contains_key(&body_context);
|
||||
if !has_selected_translation {
|
||||
continue;
|
||||
}
|
||||
let subject = translations_by_context
|
||||
.get(&subject_context)
|
||||
.map(|translation| translation.msgstr.as_str())
|
||||
.or_else(|| source_by_context.get(&subject_context).map(String::as_str))
|
||||
.with_context(|| format!("missing source subject for email template {template_key}"))?;
|
||||
let body = translations_by_context
|
||||
.get(&body_context)
|
||||
.map(|translation| translation.msgstr.as_str())
|
||||
.or_else(|| source_by_context.get(&body_context).map(String::as_str))
|
||||
.with_context(|| format!("missing source body for email template {template_key}"))?;
|
||||
template_blocks.push(format!(
|
||||
"\t{}: {{\n\t\tsubject: {},\n\t\tbody: {},\n\t}},",
|
||||
escape_identifier_or_key(template_key),
|
||||
escape_ts_single_quoted(subject),
|
||||
escape_ts_single_quoted(body)
|
||||
));
|
||||
}
|
||||
if !template_blocks.is_empty() {
|
||||
replacements.push(append_to_object_body(
|
||||
target_content,
|
||||
&target_catalog.object_body,
|
||||
template_blocks,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn split_email_context(context: &str) -> Result<(&str, &str)> {
|
||||
let Some((template, field)) = context.rsplit_once('.') else {
|
||||
bail!("email catalog context must be <template>.<subject|body>: {context}");
|
||||
};
|
||||
if field != "subject" && field != "body" {
|
||||
bail!("email catalog field must be subject or body: {context}");
|
||||
}
|
||||
Ok((template, field))
|
||||
}
|
||||
|
||||
fn append_to_object_body(
|
||||
content: &str,
|
||||
body: &Range<usize>,
|
||||
lines: Vec<String>,
|
||||
) -> (Range<usize>, String) {
|
||||
let body_content = &content[body.clone()];
|
||||
let trimmed_len = body_content.trim_end().len();
|
||||
let insert_at = body.start + trimmed_len;
|
||||
let has_existing_entries = !body_content[..trimmed_len].trim().is_empty();
|
||||
let mut insertion = String::new();
|
||||
if has_existing_entries && !body_content[..trimmed_len].trim_end().ends_with(',') {
|
||||
insertion.push(',');
|
||||
}
|
||||
insertion.push('\n');
|
||||
insertion.push_str(&lines.join("\n"));
|
||||
(insert_at..insert_at, insertion)
|
||||
}
|
||||
|
||||
fn apply_replacements(
|
||||
content: &str,
|
||||
mut replacements: Vec<(Range<usize>, String)>,
|
||||
) -> Result<String> {
|
||||
replacements.sort_by(|(left, _), (right, _)| {
|
||||
right
|
||||
.start
|
||||
.cmp(&left.start)
|
||||
.then_with(|| right.end.cmp(&left.end))
|
||||
});
|
||||
let mut rebuilt = content.to_string();
|
||||
let mut previous_start = content.len() + 1;
|
||||
for (range, replacement) in replacements {
|
||||
if range.end > previous_start {
|
||||
bail!("overlapping static TS catalog replacements");
|
||||
}
|
||||
rebuilt.replace_range(range.clone(), &replacement);
|
||||
previous_start = range.start;
|
||||
}
|
||||
Ok(rebuilt)
|
||||
}
|
||||
|
||||
fn find_const_object(content: &str, export_name: &str) -> Result<ObjectRange> {
|
||||
let export_index = content
|
||||
.find(export_name)
|
||||
.with_context(|| format!("failed to find static catalog export {export_name}"))?;
|
||||
let open_relative = content[export_index..]
|
||||
.find('{')
|
||||
.with_context(|| format!("failed to find object literal for {export_name}"))?;
|
||||
object_range_at(content, export_index + open_relative)
|
||||
}
|
||||
|
||||
fn find_call_object(content: &str, function_name: &str) -> Result<ObjectRange> {
|
||||
let mut search_start = 0;
|
||||
while let Some(relative_index) = content[search_start..].find(function_name) {
|
||||
let function_index = search_start + relative_index;
|
||||
let mut index = function_index + function_name.len();
|
||||
index = skip_ws_comments(content, index, content.len())?;
|
||||
if byte_at(content, index) != Some(b'(') {
|
||||
search_start = index;
|
||||
continue;
|
||||
}
|
||||
index = skip_ws_comments(content, index + 1, content.len())?;
|
||||
if byte_at(content, index) != Some(b'{') {
|
||||
search_start = index;
|
||||
continue;
|
||||
}
|
||||
return object_range_at(content, index);
|
||||
}
|
||||
bail!("failed to find static locale function call {function_name}");
|
||||
}
|
||||
|
||||
fn object_range_at(content: &str, open_index: usize) -> Result<ObjectRange> {
|
||||
if byte_at(content, open_index) != Some(b'{') {
|
||||
bail!("expected object literal at byte {open_index}");
|
||||
}
|
||||
let close_index = find_matching_brace(content, open_index)?;
|
||||
Ok(ObjectRange {
|
||||
body: open_index + 1..close_index,
|
||||
})
|
||||
}
|
||||
|
||||
fn find_matching_brace(content: &str, open_index: usize) -> Result<usize> {
|
||||
let bytes = content.as_bytes();
|
||||
let mut depth = 0usize;
|
||||
let mut index = open_index;
|
||||
let mut state = ScanState::Normal;
|
||||
while index < bytes.len() {
|
||||
match state {
|
||||
ScanState::Normal => match bytes[index] {
|
||||
b'\'' | b'"' | b'`' => {
|
||||
state = ScanState::String {
|
||||
quote: bytes[index],
|
||||
escaped: false,
|
||||
};
|
||||
index += 1;
|
||||
}
|
||||
b'/' if byte_at(content, index + 1) == Some(b'/') => {
|
||||
state = ScanState::LineComment;
|
||||
index += 2;
|
||||
}
|
||||
b'/' if byte_at(content, index + 1) == Some(b'*') => {
|
||||
state = ScanState::BlockComment;
|
||||
index += 2;
|
||||
}
|
||||
b'{' => {
|
||||
depth += 1;
|
||||
index += 1;
|
||||
}
|
||||
b'}' => {
|
||||
depth = depth.saturating_sub(1);
|
||||
if depth == 0 {
|
||||
return Ok(index);
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
_ => index += 1,
|
||||
},
|
||||
ScanState::String { quote, escaped } => {
|
||||
if escaped {
|
||||
state = ScanState::String {
|
||||
quote,
|
||||
escaped: false,
|
||||
};
|
||||
} else if bytes[index] == b'\\' {
|
||||
state = ScanState::String {
|
||||
quote,
|
||||
escaped: true,
|
||||
};
|
||||
} else if bytes[index] == quote {
|
||||
state = ScanState::Normal;
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
ScanState::LineComment => {
|
||||
if bytes[index] == b'\n' {
|
||||
state = ScanState::Normal;
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
ScanState::BlockComment => {
|
||||
if bytes[index] == b'*' && byte_at(content, index + 1) == Some(b'/') {
|
||||
state = ScanState::Normal;
|
||||
index += 2;
|
||||
} else {
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
bail!("unterminated object literal")
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ScanState {
|
||||
Normal,
|
||||
String { quote: u8, escaped: bool },
|
||||
LineComment,
|
||||
BlockComment,
|
||||
}
|
||||
|
||||
fn parse_string_properties(
|
||||
content: &str,
|
||||
body: &Range<usize>,
|
||||
) -> Result<Vec<ParsedStringProperty>> {
|
||||
let mut properties = Vec::new();
|
||||
let mut index = body.start;
|
||||
while index < body.end {
|
||||
index = skip_ws_comments(content, index, body.end)?;
|
||||
if index >= body.end {
|
||||
break;
|
||||
}
|
||||
if byte_at(content, index) == Some(b',') {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
let (key, next_index) = parse_property_key(content, index, body.end)?;
|
||||
index = skip_ws_comments(content, next_index, body.end)?;
|
||||
if byte_at(content, index) != Some(b':') {
|
||||
bail!("expected ':' after property key {key}");
|
||||
}
|
||||
index = skip_ws_comments(content, index + 1, body.end)?;
|
||||
let value = parse_ts_string(content, index, body.end)
|
||||
.with_context(|| format!("expected string literal value for property {key}"))?;
|
||||
index = value.value_range.end;
|
||||
properties.push(ParsedStringProperty {
|
||||
key,
|
||||
value: value.value,
|
||||
value_range: value.value_range,
|
||||
line_number: line_number_at(content, index),
|
||||
});
|
||||
}
|
||||
Ok(properties)
|
||||
}
|
||||
|
||||
fn parse_object_properties(
|
||||
content: &str,
|
||||
body: &Range<usize>,
|
||||
) -> Result<Vec<ParsedObjectProperty>> {
|
||||
let mut properties = Vec::new();
|
||||
let mut index = body.start;
|
||||
while index < body.end {
|
||||
index = skip_ws_comments(content, index, body.end)?;
|
||||
if index >= body.end {
|
||||
break;
|
||||
}
|
||||
if byte_at(content, index) == Some(b',') {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
let (key, next_index) = parse_property_key(content, index, body.end)?;
|
||||
index = skip_ws_comments(content, next_index, body.end)?;
|
||||
if byte_at(content, index) != Some(b':') {
|
||||
bail!("expected ':' after object property key {key}");
|
||||
}
|
||||
index = skip_ws_comments(content, index + 1, body.end)?;
|
||||
let object = object_range_at(content, index)
|
||||
.with_context(|| format!("expected object literal value for property {key}"))?;
|
||||
index = object.body.end + 1;
|
||||
properties.push(ParsedObjectProperty {
|
||||
key,
|
||||
body: object.body,
|
||||
});
|
||||
}
|
||||
Ok(properties)
|
||||
}
|
||||
|
||||
fn parse_property_key(content: &str, index: usize, end: usize) -> Result<(String, usize)> {
|
||||
match byte_at(content, index) {
|
||||
Some(b'\'') | Some(b'"') => {
|
||||
let parsed = parse_ts_string(content, index, end)?;
|
||||
Ok((parsed.value, parsed.value_range.end))
|
||||
}
|
||||
Some(byte) if is_identifier_start(byte) => {
|
||||
let mut next = index + 1;
|
||||
while next < end && byte_at(content, next).is_some_and(is_identifier_part) {
|
||||
next += 1;
|
||||
}
|
||||
Ok((content[index..next].to_string(), next))
|
||||
}
|
||||
_ => bail!("expected property key at byte {index}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct ParsedTsString {
|
||||
value: String,
|
||||
value_range: Range<usize>,
|
||||
}
|
||||
|
||||
fn parse_ts_string(content: &str, start: usize, end: usize) -> Result<ParsedTsString> {
|
||||
let quote = byte_at(content, start).context("expected TS string literal")?;
|
||||
if quote != b'\'' && quote != b'"' {
|
||||
bail!("expected TS string literal at byte {start}");
|
||||
}
|
||||
let mut value = String::new();
|
||||
let mut index = start + 1;
|
||||
while index < end {
|
||||
let character = content[index..]
|
||||
.chars()
|
||||
.next()
|
||||
.context("invalid TS string literal")?;
|
||||
if character as u32 == quote as u32 {
|
||||
return Ok(ParsedTsString {
|
||||
value,
|
||||
value_range: start..index + 1,
|
||||
});
|
||||
}
|
||||
if character == '\\' {
|
||||
index += 1;
|
||||
let escaped = content[index..]
|
||||
.chars()
|
||||
.next()
|
||||
.context("unterminated escape sequence in TS string literal")?;
|
||||
match escaped {
|
||||
'\'' => value.push('\''),
|
||||
'"' => value.push('"'),
|
||||
'\\' => value.push('\\'),
|
||||
'n' => value.push('\n'),
|
||||
'r' => value.push('\r'),
|
||||
't' => value.push('\t'),
|
||||
'b' => value.push('\u{0008}'),
|
||||
'f' => value.push('\u{000c}'),
|
||||
'v' => value.push('\u{000b}'),
|
||||
'0' => value.push('\0'),
|
||||
'x' => {
|
||||
let (parsed, next_index) = parse_hex_escape(content, index + 1, 2)?;
|
||||
value.push(parsed);
|
||||
index = next_index;
|
||||
continue;
|
||||
}
|
||||
'u' => {
|
||||
let (parsed, next_index) = parse_unicode_escape(content, index + 1)?;
|
||||
value.push(parsed);
|
||||
index = next_index;
|
||||
continue;
|
||||
}
|
||||
'\n' => {}
|
||||
'\r' => {}
|
||||
other => value.push(other),
|
||||
}
|
||||
index += escaped.len_utf8();
|
||||
} else {
|
||||
value.push(character);
|
||||
index += character.len_utf8();
|
||||
}
|
||||
}
|
||||
bail!("unterminated TS string literal at byte {start}")
|
||||
}
|
||||
|
||||
fn parse_hex_escape(content: &str, start: usize, len: usize) -> Result<(char, usize)> {
|
||||
let end = start + len;
|
||||
let value = u32::from_str_radix(
|
||||
content
|
||||
.get(start..end)
|
||||
.with_context(|| format!("invalid hex escape at byte {start}"))?,
|
||||
16,
|
||||
)
|
||||
.with_context(|| format!("invalid hex escape at byte {start}"))?;
|
||||
let character =
|
||||
char::from_u32(value).with_context(|| format!("invalid code point at byte {start}"))?;
|
||||
Ok((character, end))
|
||||
}
|
||||
|
||||
fn parse_unicode_escape(content: &str, start: usize) -> Result<(char, usize)> {
|
||||
if byte_at(content, start) == Some(b'{') {
|
||||
let close_relative = content[start + 1..]
|
||||
.find('}')
|
||||
.with_context(|| format!("unterminated unicode escape at byte {start}"))?;
|
||||
let digits_start = start + 1;
|
||||
let digits_end = digits_start + close_relative;
|
||||
let value = u32::from_str_radix(&content[digits_start..digits_end], 16)
|
||||
.with_context(|| format!("invalid unicode escape at byte {start}"))?;
|
||||
let character =
|
||||
char::from_u32(value).with_context(|| format!("invalid code point at byte {start}"))?;
|
||||
Ok((character, digits_end + 1))
|
||||
} else {
|
||||
parse_hex_escape(content, start, 4)
|
||||
}
|
||||
}
|
||||
|
||||
fn skip_ws_comments(content: &str, mut index: usize, end: usize) -> Result<usize> {
|
||||
while index < end {
|
||||
match byte_at(content, index) {
|
||||
Some(byte) if byte.is_ascii_whitespace() => index += 1,
|
||||
Some(b'/') if byte_at(content, index + 1) == Some(b'/') => {
|
||||
index += 2;
|
||||
while index < end && byte_at(content, index) != Some(b'\n') {
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
Some(b'/') if byte_at(content, index + 1) == Some(b'*') => {
|
||||
let close_relative = content[index + 2..end]
|
||||
.find("*/")
|
||||
.with_context(|| format!("unterminated block comment at byte {index}"))?;
|
||||
index = index + 2 + close_relative + 2;
|
||||
}
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
fn byte_at(content: &str, index: usize) -> Option<u8> {
|
||||
content.as_bytes().get(index).copied()
|
||||
}
|
||||
|
||||
fn line_number_at(content: &str, index: usize) -> usize {
|
||||
content[..index]
|
||||
.bytes()
|
||||
.filter(|byte| *byte == b'\n')
|
||||
.count()
|
||||
+ 1
|
||||
}
|
||||
|
||||
fn is_identifier_start(byte: u8) -> bool {
|
||||
byte.is_ascii_alphabetic() || byte == b'_' || byte == b'$'
|
||||
}
|
||||
|
||||
fn is_identifier_part(byte: u8) -> bool {
|
||||
is_identifier_start(byte) || byte.is_ascii_digit()
|
||||
}
|
||||
|
||||
fn is_identifier(value: &str) -> bool {
|
||||
let mut bytes = value.bytes();
|
||||
let Some(first) = bytes.next() else {
|
||||
return false;
|
||||
};
|
||||
is_identifier_start(first) && bytes.all(is_identifier_part)
|
||||
}
|
||||
|
||||
fn escape_property_key(value: &str) -> String {
|
||||
escape_ts_single_quoted(value)
|
||||
}
|
||||
|
||||
fn escape_identifier_or_key(value: &str) -> String {
|
||||
if is_identifier(value) {
|
||||
value.to_string()
|
||||
} else {
|
||||
escape_ts_single_quoted(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_ts_single_quoted(value: &str) -> String {
|
||||
let mut escaped = String::from("'");
|
||||
for character in value.chars() {
|
||||
match character {
|
||||
'\\' => escaped.push_str("\\\\"),
|
||||
'\'' => escaped.push_str("\\'"),
|
||||
'\n' => escaped.push_str("\\n"),
|
||||
'\r' => escaped.push_str("\\r"),
|
||||
'\t' => escaped.push_str("\\t"),
|
||||
'\u{2028}' => escaped.push_str("\\u2028"),
|
||||
'\u{2029}' => escaped.push_str("\\u2029"),
|
||||
other => escaped.push(other),
|
||||
}
|
||||
}
|
||||
escaped.push('\'');
|
||||
escaped
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn simple_config() -> StaticTsCatalogConfig {
|
||||
StaticTsCatalogConfig {
|
||||
kind: StaticTsCatalogKind::SimpleMessages,
|
||||
source_path: PathBuf::from("SourceMessages.ts"),
|
||||
source_export: "SOURCE_MESSAGES".to_string(),
|
||||
locale_function: "defineLocaleMessages".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn email_config() -> StaticTsCatalogConfig {
|
||||
StaticTsCatalogConfig {
|
||||
kind: StaticTsCatalogKind::EmailTemplates,
|
||||
source_path: PathBuf::from("EmailMessages.ts"),
|
||||
source_export: "EMAIL_MESSAGES".to_string(),
|
||||
locale_function: "defineEmailMessages".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_and_updates_simple_static_catalogs() {
|
||||
let source = "export const SOURCE_MESSAGES = {\n\t'hello.world': 'Hello world',\n\t'bye': 'Bye',\n} as const;\n";
|
||||
let target = "import {defineLocaleMessages} from '../Messages';\n\nexport const DE = defineLocaleMessages({\n\t'hello.world': 'Hallo Welt',\n});\n";
|
||||
let entries = read_static_ts_entries(&simple_config(), source, target, false).unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(entries[0].msgctxt.as_deref(), Some("hello.world"));
|
||||
assert_eq!(entries[0].msgstr, "Hallo Welt");
|
||||
assert_eq!(entries[1].msgstr, "");
|
||||
let rebuilt = rebuild_static_ts_allow_replacing(
|
||||
&simple_config(),
|
||||
source,
|
||||
target,
|
||||
&[
|
||||
Translation::new(Some("hello.world".to_string()), "Hello world", "Hallo"),
|
||||
Translation::new(Some("bye".to_string()), "Bye", "Tschüss"),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(rebuilt.contains("'hello.world': 'Hallo',"));
|
||||
assert!(rebuilt.contains("'bye': 'Tschüss',"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_and_updates_email_template_catalogs() {
|
||||
let source = "export const EMAIL_MESSAGES = {\n\twelcome: {\n\t\tsubject: 'Welcome to {product_name}',\n\t\tbody: 'Hello {username}\\nWelcome.',\n\t},\n\treset_password: {\n\t\tsubject: 'Reset password',\n\t\tbody: 'Use {resetUrl}',\n\t},\n} as const;\n";
|
||||
let target = "import {defineEmailMessages} from '../EmailMessages';\n\nexport const DE = defineEmailMessages({\n\twelcome: {\n\t\tsubject: 'Willkommen bei {product_name}',\n\t\tbody: 'Hallo {username}\\nWillkommen.',\n\t},\n});\n";
|
||||
let entries = read_static_ts_entries(&email_config(), source, target, false).unwrap();
|
||||
assert_eq!(entries.len(), 4);
|
||||
assert_eq!(entries[0].msgctxt.as_deref(), Some("welcome.subject"));
|
||||
assert_eq!(entries[1].msgid, "Hello {username}\nWelcome.");
|
||||
assert_eq!(entries[2].msgstr, "");
|
||||
let rebuilt = rebuild_static_ts_allow_replacing(
|
||||
&email_config(),
|
||||
source,
|
||||
target,
|
||||
&[
|
||||
Translation::new(
|
||||
Some("welcome.body".to_string()),
|
||||
"Hello {username}\nWelcome.",
|
||||
"Hallo {username}\nGuten Tag.",
|
||||
),
|
||||
Translation::new(
|
||||
Some("reset_password.subject".to_string()),
|
||||
"Reset password",
|
||||
"Passwort zurücksetzen",
|
||||
),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
assert!(rebuilt.contains("body: 'Hallo {username}\\nGuten Tag.',"));
|
||||
assert!(rebuilt.contains("reset_password: {"));
|
||||
assert!(rebuilt.contains("subject: 'Passwort zurücksetzen',"));
|
||||
assert!(rebuilt.contains("body: 'Use {resetUrl}',"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_existing_static_catalog_values() {
|
||||
let target =
|
||||
"export const DE = defineLocaleMessages({\n\t'hello.world': 'Hallo Welt',\n});\n";
|
||||
let reset = reset_static_ts_translations(&simple_config(), target).unwrap();
|
||||
assert!(reset.contains("'hello.world': '',"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"scenario": "event_log_replay_determinism",
|
||||
"seed": 4242,
|
||||
"workload": "network-partition-during-screen-share",
|
||||
"snapshot_hash": "f957df06",
|
||||
"event_count": 22,
|
||||
"final_tick": 401,
|
||||
"description": "FNV-1a hash of the simulator snapshot after the network-partition-during-screen-share scenario at seed 4242. The hash must be byte-identical across macOS, Linux, Windows. Any divergence indicates a platform-leak: floating-point determinism, HashMap iteration order, or non-deterministic clock/random.",
|
||||
"recorded_on_platform": "macos",
|
||||
"recorded_at_iso8601": "2026-06-09T00:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
schema_version = 1
|
||||
|
||||
[platforms.macos]
|
||||
description = "macOS ARM64 (this host) - real ScreenCaptureKit, CoreAudio, Metal via wgpu"
|
||||
runner = "local"
|
||||
cargo_build = true
|
||||
ts_driver = true
|
||||
scenarios = [
|
||||
"scenario_audio_capture_to_mix",
|
||||
"scenario_screen_capture_to_pool",
|
||||
"scenario_gpu_device_loss_recovery",
|
||||
"scenario_event_log_replay_determinism",
|
||||
"scenario_ffi_airlock_negative_space",
|
||||
"scenario_encoder_handoff_dryrun",
|
||||
]
|
||||
|
||||
[platforms.linux-fedora]
|
||||
description = "Fedora 42 ARM64 in Parallels VM, reached via prlctl - real PipeWire, Vulkan via wgpu"
|
||||
runner = "prlctl"
|
||||
prlctl_vm = "Fedora 42 ARM64"
|
||||
prlctl_source = "/media/psf/fluxer-dev"
|
||||
cargo_build = true
|
||||
ts_driver = true
|
||||
scenarios = [
|
||||
"scenario_audio_capture_to_mix",
|
||||
"scenario_screen_capture_to_pool",
|
||||
"scenario_gpu_device_loss_recovery",
|
||||
"scenario_event_log_replay_determinism",
|
||||
"scenario_ffi_airlock_negative_space",
|
||||
"scenario_encoder_handoff_dryrun",
|
||||
]
|
||||
|
||||
[platforms.windows-tailnet]
|
||||
description = "Windows 11 desktop on Tailnet (desktop-5ba12hj) - NVIDIA MX450 + Intel Iris Xe, WGC, DXGI, D3D11, NVENC, QSV"
|
||||
runner = "ssh"
|
||||
ssh_host = "desktop-5ba12hj"
|
||||
ssh_user = "Hampus"
|
||||
cargo_build = true
|
||||
ts_driver = true
|
||||
scenarios = [
|
||||
"scenario_audio_capture_to_mix",
|
||||
"scenario_screen_capture_to_pool",
|
||||
"scenario_gpu_device_loss_recovery",
|
||||
"scenario_event_log_replay_determinism",
|
||||
"scenario_ffi_airlock_negative_space",
|
||||
"scenario_encoder_handoff_dryrun",
|
||||
]
|
||||
|
||||
[platforms.windows-vm]
|
||||
description = "Windows 11 ARM Parallels VM - ARM-only, limited GPU access"
|
||||
runner = "prlctl"
|
||||
prlctl_vm = "Windows 11"
|
||||
cargo_build = true
|
||||
ts_driver = true
|
||||
scenarios = [
|
||||
"scenario_audio_capture_to_mix",
|
||||
"scenario_screen_capture_to_pool",
|
||||
"scenario_event_log_replay_determinism",
|
||||
"scenario_ffi_airlock_negative_space",
|
||||
"scenario_encoder_handoff_dryrun",
|
||||
]
|
||||
|
||||
[scenario.scenario_audio_capture_to_mix]
|
||||
description = "Real PCM capture -> SourceRing -> AudioMixSession -> APM -> mix output"
|
||||
expected_runtime_seconds = 5
|
||||
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
|
||||
driver = "rust"
|
||||
subcommand = "audio-capture-to-mix"
|
||||
|
||||
[scenario.scenario_screen_capture_to_pool]
|
||||
description = "Real screen capture -> FramePool with allocation-free steady state"
|
||||
expected_runtime_seconds = 5
|
||||
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
|
||||
driver = "rust"
|
||||
subcommand = "screen-capture-to-pool"
|
||||
|
||||
[scenario.scenario_gpu_device_loss_recovery]
|
||||
description = "wgpu device loss -> GpuLossRegistry rebuild walk -> Nv12Packer ready"
|
||||
expected_runtime_seconds = 10
|
||||
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet"]
|
||||
driver = "rust"
|
||||
subcommand = "gpu-device-loss-recovery"
|
||||
|
||||
[scenario.scenario_event_log_replay_determinism]
|
||||
description = "Fixed-seed simulator scenario - hash agreement across platforms"
|
||||
expected_runtime_seconds = 20
|
||||
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
|
||||
driver = "typescript"
|
||||
ts_entry = "tools/integration/ts-driver/event_log_replay_determinism.ts"
|
||||
|
||||
[scenario.scenario_ffi_airlock_negative_space]
|
||||
description = "Malformed audio frame -> Rust receive-side assert panic with expected message"
|
||||
expected_runtime_seconds = 3
|
||||
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
|
||||
driver = "rust"
|
||||
subcommand = "ffi-airlock-negative"
|
||||
|
||||
[scenario.scenario_encoder_handoff_dryrun]
|
||||
description = "EncoderInputRing - D3D11KeyedMutexBackend on Windows, CpuMemcpyBackend elsewhere - 8 slots, DTS offset, skip-don't-block"
|
||||
expected_runtime_seconds = 5
|
||||
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
|
||||
driver = "rust"
|
||||
subcommand = "encoder-handoff-dryrun"
|
||||
@@ -0,0 +1,3 @@
|
||||
target/
|
||||
target-*/
|
||||
Cargo.lock
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "fluxer_integration_driver"
|
||||
version = "0.0.0"
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0-or-later"
|
||||
publish = false
|
||||
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
|
||||
[[bin]]
|
||||
name = "integration-driver"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
fluxer_screen_frame_bus = {path = "../../../fluxer_desktop/native/screen-frame-bus", features = ["wgpu"]}
|
||||
fluxer_gpu_rebuild = {path = "../../../fluxer_desktop/native/gpu-rebuild", features = ["wgpu"]}
|
||||
fluxer_audio_mix = {path = "../../../fluxer_desktop/native/audio-mix"}
|
||||
fluxer_audio_apm = {path = "../../../fluxer_desktop/native/audio-apm"}
|
||||
fluxer_nv12_gpu_pack = {path = "../../../fluxer_desktop/native/nv12-gpu-pack"}
|
||||
fluxer_encoder_ring = {path = "../../../fluxer_desktop/native/encoder-ring"}
|
||||
fluxer_rt_thread = {path = "../../../fluxer_desktop/native/rt-thread"}
|
||||
wgpu = "29"
|
||||
serde = {version = "1", features = ["derive"]}
|
||||
serde_json = "1"
|
||||
@@ -0,0 +1,50 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
#![deny(warnings)]
|
||||
#![deny(clippy::unwrap_used)]
|
||||
#![deny(clippy::panic)]
|
||||
#![deny(clippy::too_many_lines)]
|
||||
|
||||
mod scenarios;
|
||||
|
||||
use std::process::ExitCode;
|
||||
|
||||
const USAGE: &str = "usage: integration-driver <scenario> [args...]\n\
|
||||
scenarios:\n \
|
||||
audio-capture-to-mix\n \
|
||||
screen-capture-to-pool\n \
|
||||
gpu-device-loss-recovery\n \
|
||||
encoder-handoff-dryrun\n \
|
||||
ffi-airlock-negative [invalid-sample-rate|valid-baseline]\n";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
assert!(!args.is_empty(), "argv must contain program name");
|
||||
if args.len() < 2 {
|
||||
eprintln!("{}", USAGE);
|
||||
return ExitCode::from(64);
|
||||
}
|
||||
let scenario = args[1].as_str();
|
||||
let tail: Vec<&str> = args[2..].iter().map(String::as_str).collect();
|
||||
let result = match scenario {
|
||||
"audio-capture-to-mix" => scenarios::audio_capture_to_mix::run(&tail),
|
||||
"screen-capture-to-pool" => scenarios::screen_capture_to_pool::run(&tail),
|
||||
"gpu-device-loss-recovery" => scenarios::gpu_device_loss_recovery::run(&tail),
|
||||
"encoder-handoff-dryrun" => scenarios::encoder_handoff_dryrun::run(&tail),
|
||||
"ffi-airlock-negative" => scenarios::ffi_airlock_negative::run(&tail),
|
||||
other => {
|
||||
eprintln!("unknown scenario {other}\n{USAGE}");
|
||||
return ExitCode::from(64);
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Ok(report) => {
|
||||
println!("{}", report);
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
Err(report) => {
|
||||
println!("{}", report);
|
||||
ExitCode::from(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use fluxer_audio_apm::{ApmConfigBuilder, AudioProcessor, StubAudioProcessor};
|
||||
use fluxer_audio_mix::{AUDIO_OUTPUT_FRAMES, AudioMixSession, SourceRing};
|
||||
use fluxer_rt_thread::TickInfo;
|
||||
use serde_json::json;
|
||||
|
||||
use super::ScenarioReport;
|
||||
|
||||
const TONE_AMPLITUDE: i16 = 8_000;
|
||||
const TONE_SAMPLE_RATE_HZ: u32 = 48_000;
|
||||
const TONE_FREQUENCY_HZ: u32 = 440;
|
||||
const TONE_TICK_COUNT: u64 = 4;
|
||||
const APM_SAMPLE_RATE_HZ: u32 = 48_000;
|
||||
const APM_CHANNELS: u16 = 1;
|
||||
const APM_FRAME_SAMPLES: usize = 480;
|
||||
|
||||
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
|
||||
assert_eq!(APM_FRAME_SAMPLES, 480, "APM 10 ms @ 48 kHz invariant");
|
||||
let measurements = match drive_pipeline() {
|
||||
Ok(m) => m,
|
||||
Err(reason) => {
|
||||
return Err(ScenarioReport::fail(
|
||||
"audio_capture_to_mix",
|
||||
json!({"reason": reason}),
|
||||
Vec::new(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let mut assertions = Vec::new();
|
||||
if measurements.tone_mix_peak < (TONE_AMPLITUDE as i64) / 4 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"audio_capture_to_mix",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec!["tone peak below quarter-amplitude floor".to_string()],
|
||||
));
|
||||
}
|
||||
assertions.push("tone present in mix output above quarter-amplitude floor".to_string());
|
||||
if measurements.apm_capture_frames_processed == 0 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"audio_capture_to_mix",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec!["APM processed zero capture frames".to_string()],
|
||||
));
|
||||
}
|
||||
assertions.push("APM processed at least one capture frame".to_string());
|
||||
if measurements.mix_ticks_completed != TONE_TICK_COUNT {
|
||||
return Err(ScenarioReport::fail(
|
||||
"audio_capture_to_mix",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec![format!(
|
||||
"expected {} mix ticks, observed {}",
|
||||
TONE_TICK_COUNT, measurements.mix_ticks_completed
|
||||
)],
|
||||
));
|
||||
}
|
||||
assertions.push("mix session completed the full tick schedule".to_string());
|
||||
Ok(ScenarioReport::pass(
|
||||
"audio_capture_to_mix",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
assertions,
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct Measurements {
|
||||
tone_mix_peak: i64,
|
||||
tone_mix_rms: i64,
|
||||
mix_ticks_completed: u64,
|
||||
apm_capture_frames_processed: u64,
|
||||
source_ring_pushed_total: u64,
|
||||
source_ring_drained_total: u64,
|
||||
sample_rate_hz: u32,
|
||||
tone_frequency_hz: u32,
|
||||
}
|
||||
|
||||
fn drive_pipeline() -> Result<Measurements, String> {
|
||||
let (mut producer, consumer) = SourceRing::create(8192, TONE_SAMPLE_RATE_HZ)
|
||||
.map_err(|err| format!("ring create failed: {err:?}"))?;
|
||||
let pushed_per_tick = AUDIO_OUTPUT_FRAMES;
|
||||
let mut phase: u32 = 0;
|
||||
let mut apm_buffer: Vec<i16> = vec![0i16; APM_FRAME_SAMPLES];
|
||||
let apm_config = ApmConfigBuilder::new()
|
||||
.aec(false)
|
||||
.ns(false)
|
||||
.agc(false)
|
||||
.build();
|
||||
let mut apm = StubAudioProcessor::new(apm_config, APM_SAMPLE_RATE_HZ, APM_CHANNELS)
|
||||
.map_err(|err| format!("apm init failed: {err:?}"))?;
|
||||
let mut source_ring_pushed_total: u64 = 0;
|
||||
for _ in 0..TONE_TICK_COUNT {
|
||||
let tone = generate_tone_chunk(pushed_per_tick, &mut phase);
|
||||
assert_eq!(tone.len(), pushed_per_tick, "tone chunk length");
|
||||
let mut apm_offset: usize = 0;
|
||||
while apm_offset + APM_FRAME_SAMPLES <= tone.len() {
|
||||
apm_buffer.copy_from_slice(&tone[apm_offset..apm_offset + APM_FRAME_SAMPLES]);
|
||||
let _ = apm
|
||||
.process_capture_frame(&mut apm_buffer, APM_SAMPLE_RATE_HZ, APM_CHANNELS)
|
||||
.map_err(|err| format!("apm process failed: {err:?}"))?;
|
||||
apm_offset += APM_FRAME_SAMPLES;
|
||||
}
|
||||
let pushed = producer.try_push_slice(&tone);
|
||||
assert_eq!(pushed, tone.len(), "ring must accept full tone chunk");
|
||||
source_ring_pushed_total = source_ring_pushed_total.saturating_add(pushed as u64);
|
||||
}
|
||||
let mut session = AudioMixSession::new(vec![consumer], AUDIO_OUTPUT_FRAMES)
|
||||
.map_err(|err| format!("mix init failed: {err:?}"))?;
|
||||
let mut peak_observed: i64 = 0;
|
||||
let mut accumulator_sq: u64 = 0;
|
||||
let mut counted: u64 = 0;
|
||||
for tick_index in 0..TONE_TICK_COUNT {
|
||||
let tick = synthetic_tick(tick_index);
|
||||
let _ = session.tick(tick);
|
||||
let output = session.last_output();
|
||||
for sample in output.iter() {
|
||||
let abs = (*sample as i64).abs();
|
||||
if abs > peak_observed {
|
||||
peak_observed = abs;
|
||||
}
|
||||
let sq = (*sample as i64).saturating_mul(*sample as i64);
|
||||
accumulator_sq = accumulator_sq.saturating_add(sq as u64);
|
||||
counted = counted.saturating_add(1);
|
||||
}
|
||||
}
|
||||
let rms = accumulator_sq.checked_div(counted).unwrap_or(0).isqrt() as i64;
|
||||
Ok(Measurements {
|
||||
tone_mix_peak: peak_observed,
|
||||
tone_mix_rms: rms,
|
||||
mix_ticks_completed: session.ticks_completed(),
|
||||
apm_capture_frames_processed: apm.capture_frames_processed(),
|
||||
source_ring_pushed_total,
|
||||
source_ring_drained_total: 0,
|
||||
sample_rate_hz: TONE_SAMPLE_RATE_HZ,
|
||||
tone_frequency_hz: TONE_FREQUENCY_HZ,
|
||||
})
|
||||
}
|
||||
|
||||
fn generate_tone_chunk(samples: usize, phase: &mut u32) -> Vec<i16> {
|
||||
assert!(samples > 0, "tone chunk samples positive");
|
||||
assert!(samples <= 1 << 14, "tone chunk within sanity cap");
|
||||
let mut out = Vec::with_capacity(samples);
|
||||
let period_samples = TONE_SAMPLE_RATE_HZ / TONE_FREQUENCY_HZ;
|
||||
assert!(period_samples > 0, "period must be positive");
|
||||
for _ in 0..samples {
|
||||
let progress = (*phase as f32) / (period_samples as f32);
|
||||
let radians = progress * std::f32::consts::TAU;
|
||||
let value = (radians.sin() * TONE_AMPLITUDE as f32) as i16;
|
||||
out.push(value);
|
||||
*phase = phase.wrapping_add(1);
|
||||
if *phase >= period_samples {
|
||||
*phase = 0;
|
||||
}
|
||||
}
|
||||
assert_eq!(out.len(), samples, "tone chunk length post-condition");
|
||||
out
|
||||
}
|
||||
|
||||
fn synthetic_tick(index: u64) -> TickInfo {
|
||||
let scheduled_ns = index.saturating_mul(21_333_333);
|
||||
TickInfo {
|
||||
tick_index: index,
|
||||
scheduled_ns,
|
||||
actual_ns: scheduled_ns,
|
||||
lag_ns: 0,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use fluxer_encoder_ring::{
|
||||
CpuMemcpyBackend, EncoderInputRing, RingError, TextureFormat, apply_dts_offset,
|
||||
compute_dts_offset_us,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
use super::ScenarioReport;
|
||||
|
||||
const FRAME_WIDTH_PX: u32 = 1280;
|
||||
const FRAME_HEIGHT_PX: u32 = 720;
|
||||
const RING_DEPTH_TARGET: u32 = 8;
|
||||
const LAG_OVERSUBSCRIBE_COUNT: u32 = 4;
|
||||
const FRAME_INTERVAL_US: u64 = 16_666;
|
||||
const NUM_B_FRAMES: u32 = 2;
|
||||
const FIRST_PTS_US: u64 = 0;
|
||||
|
||||
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
|
||||
let measurements = match drive_ring() {
|
||||
Ok(m) => m,
|
||||
Err(reason) => {
|
||||
return Err(ScenarioReport::fail(
|
||||
"encoder_handoff_dryrun",
|
||||
json!({"reason": reason}),
|
||||
Vec::new(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let mut assertions = Vec::new();
|
||||
if measurements.completed_count != RING_DEPTH_TARGET as u64 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"encoder_handoff_dryrun",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec![format!(
|
||||
"expected {} completed slots, observed {}",
|
||||
RING_DEPTH_TARGET, measurements.completed_count
|
||||
)],
|
||||
));
|
||||
}
|
||||
assertions.push("eight slots submitted and completed".to_string());
|
||||
if measurements.dropped_count != LAG_OVERSUBSCRIBE_COUNT as u64 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"encoder_handoff_dryrun",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec![format!(
|
||||
"expected {} dropped under oversubscription, observed {}",
|
||||
LAG_OVERSUBSCRIBE_COUNT, measurements.dropped_count
|
||||
)],
|
||||
));
|
||||
}
|
||||
assertions.push("oversubscribed submits report skip-don't-block drop counter".to_string());
|
||||
if measurements.dispatched_count != RING_DEPTH_TARGET as u64 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"encoder_handoff_dryrun",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec![format!(
|
||||
"expected {} dispatched ready frames, observed {}",
|
||||
RING_DEPTH_TARGET, measurements.dispatched_count
|
||||
)],
|
||||
));
|
||||
}
|
||||
assertions.push("dispatched_count equals submitted_count in steady state".to_string());
|
||||
if measurements.dts_first_frame > measurements.pts_first_frame {
|
||||
return Err(ScenarioReport::fail(
|
||||
"encoder_handoff_dryrun",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec!["first DTS must not exceed PTS".to_string()],
|
||||
));
|
||||
}
|
||||
assertions.push("DTS offset is non-positive given B-frames".to_string());
|
||||
Ok(ScenarioReport::pass(
|
||||
"encoder_handoff_dryrun",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
assertions,
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct Measurements {
|
||||
backend: &'static str,
|
||||
submitted_count: u64,
|
||||
completed_count: u64,
|
||||
dispatched_count: u64,
|
||||
dropped_count: u64,
|
||||
capacity: usize,
|
||||
pts_first_frame: u64,
|
||||
dts_first_frame: u64,
|
||||
dts_offset_us: i64,
|
||||
texture_skipped_frames: u32,
|
||||
}
|
||||
|
||||
fn drive_ring() -> Result<Measurements, String> {
|
||||
let backend = CpuMemcpyBackend::new();
|
||||
let mut ring: EncoderInputRing<CpuMemcpyBackend> = EncoderInputRing::new(backend);
|
||||
ring.initialise(FRAME_WIDTH_PX, FRAME_HEIGHT_PX, TextureFormat::Nv12)
|
||||
.map_err(|err| format!("ring init failed: {err:?}"))?;
|
||||
assert_eq!(
|
||||
ring.capacity(),
|
||||
RING_DEPTH_TARGET as usize,
|
||||
"ring capacity invariant"
|
||||
);
|
||||
assert_eq!(
|
||||
ring.free_count(),
|
||||
RING_DEPTH_TARGET as usize,
|
||||
"fresh ring fully free"
|
||||
);
|
||||
for _ in 0..RING_DEPTH_TARGET {
|
||||
ring.submit(noop_fill)
|
||||
.map_err(|err| format!("first round submit failed: {err:?}"))?;
|
||||
}
|
||||
let mut texture_skipped_frames: u32 = 0;
|
||||
for _ in 0..LAG_OVERSUBSCRIBE_COUNT {
|
||||
match ring.submit(noop_fill) {
|
||||
Err(RingError::FullDropped { .. }) => {
|
||||
texture_skipped_frames = texture_skipped_frames.saturating_add(1);
|
||||
}
|
||||
Err(err) => return Err(format!("unexpected submit error: {err:?}")),
|
||||
Ok(()) => return Err("oversubscribed submit must drop".to_string()),
|
||||
}
|
||||
}
|
||||
let dts_offset_us = compute_dts_offset_us(FIRST_PTS_US, NUM_B_FRAMES, FRAME_INTERVAL_US);
|
||||
assert!(dts_offset_us <= 0, "dts offset non-positive");
|
||||
let pts_first_frame = FRAME_INTERVAL_US.saturating_mul(NUM_B_FRAMES as u64);
|
||||
let dts_first_frame = apply_dts_offset(pts_first_frame, dts_offset_us);
|
||||
let mut released_sequences: Vec<u64> = Vec::with_capacity(RING_DEPTH_TARGET as usize);
|
||||
while let Some(ready) = ring.poll_next_ready() {
|
||||
released_sequences.push(ready.sequence);
|
||||
ring.release_completed(ready)
|
||||
.map_err(|err| format!("release failed: {err:?}"))?;
|
||||
}
|
||||
assert_eq!(
|
||||
released_sequences.len(),
|
||||
RING_DEPTH_TARGET as usize,
|
||||
"expected all submitted frames to drain"
|
||||
);
|
||||
for (idx, seq) in released_sequences.iter().enumerate() {
|
||||
let expected = (idx + 1) as u64;
|
||||
assert_eq!(*seq, expected, "FIFO release order");
|
||||
}
|
||||
let metrics = ring.metrics();
|
||||
Ok(Measurements {
|
||||
backend: backend_label(),
|
||||
submitted_count: metrics.submitted_count,
|
||||
completed_count: metrics.completed_count,
|
||||
dispatched_count: metrics.dispatched_count,
|
||||
dropped_count: metrics.dropped_count,
|
||||
capacity: ring.capacity(),
|
||||
pts_first_frame,
|
||||
dts_first_frame,
|
||||
dts_offset_us,
|
||||
texture_skipped_frames,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
const fn backend_label() -> &'static str {
|
||||
"cpu_memcpy_with_d3d11_keyed_mutex_available"
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
const fn backend_label() -> &'static str {
|
||||
"cpu_memcpy_backend"
|
||||
}
|
||||
|
||||
fn noop_fill(_slot: &mut fluxer_encoder_ring::CpuSlotHandle) {}
|
||||
@@ -0,0 +1,234 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use serde_json::json;
|
||||
use std::fmt;
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
|
||||
use super::ScenarioReport;
|
||||
|
||||
const VALID_SAMPLE_RATE_HZ: u32 = 48_000;
|
||||
const INVALID_SAMPLE_RATE_HZ: u32 = 44_100;
|
||||
const VALID_CHANNELS: u32 = 1;
|
||||
const VALID_FRAME_BYTES: u32 = 960;
|
||||
const VALID_TIMESTAMP_NS: u64 = 1_000_000;
|
||||
const AUDIO_FRAME_BYTES_MAX: u32 = 1 << 20;
|
||||
const AUDIO_SAMPLE_RATES_HZ: [u32; 3] = [16_000, 32_000, 48_000];
|
||||
const AUDIO_CHANNELS_VALID: [u32; 2] = [1, 2];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct VoiceEngineV2AudioFrameInvariants {
|
||||
sample_rate_hz: u32,
|
||||
num_channels: u32,
|
||||
frame_bytes: u32,
|
||||
timestamp_ns: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum VoiceEngineV2AssertError {
|
||||
FrameBytesOutOfRange { bytes: u32, max: u32 },
|
||||
SampleRateInvalid { hz: u32 },
|
||||
ChannelsInvalid { channels: u32 },
|
||||
TimestampRegressed { previous_ns: u64, received_ns: u64 },
|
||||
}
|
||||
|
||||
impl fmt::Display for VoiceEngineV2AssertError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::FrameBytesOutOfRange { bytes, max } => {
|
||||
write!(
|
||||
formatter,
|
||||
"AudioFrameBytesOutOfRange: audio frame bytes {bytes} not in (0, {max}]"
|
||||
)
|
||||
}
|
||||
Self::SampleRateInvalid { hz } => {
|
||||
write!(
|
||||
formatter,
|
||||
"AudioSampleRateInvalid: audio sample rate {hz} not in [16000, 32000, 48000]"
|
||||
)
|
||||
}
|
||||
Self::ChannelsInvalid { channels } => {
|
||||
write!(
|
||||
formatter,
|
||||
"AudioChannelsInvalid: audio channels {channels} not in [1, 2]"
|
||||
)
|
||||
}
|
||||
Self::TimestampRegressed {
|
||||
previous_ns,
|
||||
received_ns,
|
||||
} => {
|
||||
write!(
|
||||
formatter,
|
||||
"AudioTimestampRegressed: audio timestamp {received_ns} did not exceed previous {previous_ns}"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_audio_frame_invariants(
|
||||
frame: VoiceEngineV2AudioFrameInvariants,
|
||||
previous_timestamp_ns: Option<u64>,
|
||||
) -> Result<(), VoiceEngineV2AssertError> {
|
||||
if frame.frame_bytes == 0 || frame.frame_bytes > AUDIO_FRAME_BYTES_MAX {
|
||||
return Err(VoiceEngineV2AssertError::FrameBytesOutOfRange {
|
||||
bytes: frame.frame_bytes,
|
||||
max: AUDIO_FRAME_BYTES_MAX,
|
||||
});
|
||||
}
|
||||
if !AUDIO_SAMPLE_RATES_HZ.contains(&frame.sample_rate_hz) {
|
||||
return Err(VoiceEngineV2AssertError::SampleRateInvalid {
|
||||
hz: frame.sample_rate_hz,
|
||||
});
|
||||
}
|
||||
if !AUDIO_CHANNELS_VALID.contains(&frame.num_channels) {
|
||||
return Err(VoiceEngineV2AssertError::ChannelsInvalid {
|
||||
channels: frame.num_channels,
|
||||
});
|
||||
}
|
||||
if let Some(previous_ns) = previous_timestamp_ns
|
||||
&& frame.timestamp_ns <= previous_ns
|
||||
{
|
||||
return Err(VoiceEngineV2AssertError::TimestampRegressed {
|
||||
previous_ns,
|
||||
received_ns: frame.timestamp_ns,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::panic)]
|
||||
fn assert_audio_frame_invariants(
|
||||
frame: VoiceEngineV2AudioFrameInvariants,
|
||||
previous_timestamp_ns: Option<u64>,
|
||||
) {
|
||||
let result = check_audio_frame_invariants(frame, previous_timestamp_ns);
|
||||
if let Err(err) = result {
|
||||
panic!("{err}");
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
|
||||
let prior_hook = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(|_info| {}));
|
||||
let result = run_inner(args);
|
||||
std::panic::set_hook(prior_hook);
|
||||
result
|
||||
}
|
||||
|
||||
fn run_inner(args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
|
||||
let mode = args.first().copied().unwrap_or("invalid-sample-rate");
|
||||
let measurements = match mode {
|
||||
"invalid-sample-rate" => exercise_invalid_sample_rate(),
|
||||
"valid-baseline" => exercise_valid_baseline(),
|
||||
other => {
|
||||
return Err(ScenarioReport::fail(
|
||||
"ffi_airlock_negative",
|
||||
json!({"reason": format!("unknown mode: {other}")}),
|
||||
Vec::new(),
|
||||
));
|
||||
}
|
||||
};
|
||||
match measurements {
|
||||
Ok(m) => Ok(ScenarioReport::pass(
|
||||
"ffi_airlock_negative",
|
||||
serde_json::to_value(&m).unwrap_or(json!({})),
|
||||
m.assertions,
|
||||
)),
|
||||
Err(report) => Err(report),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct Measurements {
|
||||
mode: &'static str,
|
||||
panic_observed: bool,
|
||||
panic_message: Option<String>,
|
||||
valid_baseline_passed: bool,
|
||||
#[serde(skip_serializing)]
|
||||
assertions: Vec<String>,
|
||||
}
|
||||
|
||||
fn exercise_invalid_sample_rate() -> Result<Measurements, ScenarioReport> {
|
||||
let frame = VoiceEngineV2AudioFrameInvariants {
|
||||
sample_rate_hz: INVALID_SAMPLE_RATE_HZ,
|
||||
num_channels: VALID_CHANNELS,
|
||||
frame_bytes: VALID_FRAME_BYTES,
|
||||
timestamp_ns: VALID_TIMESTAMP_NS,
|
||||
};
|
||||
let outcome = catch_unwind(AssertUnwindSafe(|| {
|
||||
assert_audio_frame_invariants(frame, None);
|
||||
}));
|
||||
match outcome {
|
||||
Ok(()) => Err(ScenarioReport::fail(
|
||||
"ffi_airlock_negative",
|
||||
json!({"mode": "invalid-sample-rate", "panic_observed": false}),
|
||||
vec!["receive-side assertion failed to panic on invalid sample rate".to_string()],
|
||||
)),
|
||||
Err(payload) => {
|
||||
let message = panic_payload_to_string(&payload);
|
||||
if !message.contains("AudioSampleRateInvalid")
|
||||
&& !message.contains("not in [16000, 32000, 48000]")
|
||||
{
|
||||
return Err(ScenarioReport::fail(
|
||||
"ffi_airlock_negative",
|
||||
json!({"mode": "invalid-sample-rate", "panic_observed": true, "panic_message": message}),
|
||||
vec!["panic message did not mention AudioSampleRateInvalid".to_string()],
|
||||
));
|
||||
}
|
||||
let assertions = vec![
|
||||
"receive-side assertion panicked as expected".to_string(),
|
||||
"panic message names AudioSampleRateInvalid".to_string(),
|
||||
];
|
||||
Ok(Measurements {
|
||||
mode: "invalid-sample-rate",
|
||||
panic_observed: true,
|
||||
panic_message: Some(message),
|
||||
valid_baseline_passed: false,
|
||||
assertions,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn exercise_valid_baseline() -> Result<Measurements, ScenarioReport> {
|
||||
let frame = VoiceEngineV2AudioFrameInvariants {
|
||||
sample_rate_hz: VALID_SAMPLE_RATE_HZ,
|
||||
num_channels: VALID_CHANNELS,
|
||||
frame_bytes: VALID_FRAME_BYTES,
|
||||
timestamp_ns: VALID_TIMESTAMP_NS,
|
||||
};
|
||||
let outcome = catch_unwind(AssertUnwindSafe(|| {
|
||||
assert_audio_frame_invariants(frame, None);
|
||||
}));
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
let assertions = vec!["valid-frame baseline did not panic".to_string()];
|
||||
Ok(Measurements {
|
||||
mode: "valid-baseline",
|
||||
panic_observed: false,
|
||||
panic_message: None,
|
||||
valid_baseline_passed: true,
|
||||
assertions,
|
||||
})
|
||||
}
|
||||
Err(payload) => Err(ScenarioReport::fail(
|
||||
"ffi_airlock_negative",
|
||||
json!({
|
||||
"mode": "valid-baseline",
|
||||
"panic_observed": true,
|
||||
"panic_message": panic_payload_to_string(&payload),
|
||||
}),
|
||||
vec!["valid baseline frame must not panic".to_string()],
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn panic_payload_to_string(payload: &Box<dyn std::any::Any + Send>) -> String {
|
||||
if let Some(s) = payload.downcast_ref::<&'static str>() {
|
||||
return (*s).to_string();
|
||||
}
|
||||
if let Some(s) = payload.downcast_ref::<String>() {
|
||||
return s.clone();
|
||||
}
|
||||
"<non-string panic payload>".to_string()
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use fluxer_gpu_rebuild::{GpuLossCallback, GpuLossRegistry};
|
||||
use fluxer_nv12_gpu_pack::Nv12Packer;
|
||||
use fluxer_screen_frame_bus::gpu_loss::{
|
||||
WgpuStagingBackend, WgpuStagingConfig, try_acquire_device,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
use super::ScenarioReport;
|
||||
|
||||
const PACKER_MAX_WIDTH: u32 = 1280;
|
||||
const PACKER_MAX_HEIGHT: u32 = 720;
|
||||
const STAGING_BYTE_LEN: u64 = 64 * 1024;
|
||||
|
||||
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
|
||||
let (device, queue, _instance) = acquire_gpu_context()?;
|
||||
let measurements = drive_rebuild(&device, &queue).map_err(|reason| {
|
||||
ScenarioReport::fail(
|
||||
"gpu_device_loss_recovery",
|
||||
json!({"reason": reason}),
|
||||
Vec::new(),
|
||||
)
|
||||
})?;
|
||||
let assertions = check_measurements(&measurements)?;
|
||||
Ok(ScenarioReport::pass(
|
||||
"gpu_device_loss_recovery",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
assertions,
|
||||
))
|
||||
}
|
||||
|
||||
fn acquire_gpu_context() -> Result<(wgpu::Device, wgpu::Queue, wgpu::Instance), ScenarioReport> {
|
||||
let acquired = std::panic::catch_unwind(std::panic::AssertUnwindSafe(try_acquire_device));
|
||||
match acquired {
|
||||
Ok(Some(triple)) => Ok(triple),
|
||||
Ok(None) => Err(ScenarioReport::fail(
|
||||
"gpu_device_loss_recovery",
|
||||
json!({"reason": "no wgpu adapter available"}),
|
||||
Vec::new(),
|
||||
)),
|
||||
Err(_) => Err(ScenarioReport::fail(
|
||||
"gpu_device_loss_recovery",
|
||||
json!({"reason": "wgpu adapter acquisition panicked"}),
|
||||
Vec::new(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn check_measurements(measurements: &Measurements) -> Result<Vec<String>, ScenarioReport> {
|
||||
if measurements.released_count != 2 {
|
||||
return Err(fail_with(
|
||||
measurements,
|
||||
format!(
|
||||
"expected 2 release calls, observed {}",
|
||||
measurements.released_count
|
||||
),
|
||||
));
|
||||
}
|
||||
if measurements.rebuilt_count != 2 {
|
||||
return Err(fail_with(
|
||||
measurements,
|
||||
format!(
|
||||
"expected 2 rebuilt owners, observed {}",
|
||||
measurements.rebuilt_count
|
||||
),
|
||||
));
|
||||
}
|
||||
if measurements.failed_count != 0 {
|
||||
return Err(fail_with(
|
||||
measurements,
|
||||
format!(
|
||||
"expected 0 failures, observed {}",
|
||||
measurements.failed_count
|
||||
),
|
||||
));
|
||||
}
|
||||
if !measurements.packer_ready_after_rebuild {
|
||||
return Err(fail_with(
|
||||
measurements,
|
||||
"Nv12Packer not ready after rebuild".to_string(),
|
||||
));
|
||||
}
|
||||
if !measurements.staging_ready_after_rebuild {
|
||||
return Err(fail_with(
|
||||
measurements,
|
||||
"WgpuStagingBackend not ready after rebuild".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(vec![
|
||||
"registry released both owners".to_string(),
|
||||
"registry rebuilt both owners".to_string(),
|
||||
"zero failures during rebuild walk".to_string(),
|
||||
"Nv12Packer reports is_ready true after rebuild".to_string(),
|
||||
"WgpuStagingBackend reports is_ready true after rebuild".to_string(),
|
||||
])
|
||||
}
|
||||
|
||||
fn fail_with(measurements: &Measurements, message: String) -> ScenarioReport {
|
||||
ScenarioReport::fail(
|
||||
"gpu_device_loss_recovery",
|
||||
serde_json::to_value(measurements).unwrap_or(json!({})),
|
||||
vec![message],
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct Measurements {
|
||||
released_count: u32,
|
||||
rebuilt_count: u32,
|
||||
failed_count: u32,
|
||||
vacant_count: u32,
|
||||
packer_ready_after_rebuild: bool,
|
||||
staging_ready_after_rebuild: bool,
|
||||
adapter_info: String,
|
||||
}
|
||||
|
||||
fn drive_rebuild(device: &wgpu::Device, queue: &wgpu::Queue) -> Result<Measurements, String> {
|
||||
let registry = GpuLossRegistry::new();
|
||||
let packer = Box::new(Nv12Packer::new(device, PACKER_MAX_WIDTH, PACKER_MAX_HEIGHT));
|
||||
let staging = Box::new(WgpuStagingBackend::new(
|
||||
device,
|
||||
WgpuStagingConfig::new(STAGING_BYTE_LEN),
|
||||
));
|
||||
let packer_built_before = GpuLossCallback::is_ready(packer.as_ref());
|
||||
let staging_built_before = GpuLossCallback::is_ready(staging.as_ref());
|
||||
assert!(packer_built_before, "packer must be ready before rebuild");
|
||||
assert!(staging_built_before, "staging must be ready before rebuild");
|
||||
let _packer_guard = registry.register(packer);
|
||||
let _staging_guard = registry.register(staging);
|
||||
let report = registry.handle_device_lost(device, queue);
|
||||
assert!(
|
||||
report.released_count as usize <= 2,
|
||||
"no more than two owners released"
|
||||
);
|
||||
let mut packer_ready = false;
|
||||
let mut staging_ready = false;
|
||||
for outcome in report.outcomes.iter() {
|
||||
match outcome {
|
||||
fluxer_gpu_rebuild::RebuildOutcome::Rebuilt { label, .. } => {
|
||||
if label.contains("nv12") {
|
||||
packer_ready = true;
|
||||
}
|
||||
if label.contains("staging") {
|
||||
staging_ready = true;
|
||||
}
|
||||
}
|
||||
fluxer_gpu_rebuild::RebuildOutcome::Failed { .. } => {
|
||||
return Err("rebuild outcome reported failure".to_string());
|
||||
}
|
||||
fluxer_gpu_rebuild::RebuildOutcome::Vacant { .. } => {}
|
||||
}
|
||||
}
|
||||
Ok(Measurements {
|
||||
released_count: report.released_count,
|
||||
rebuilt_count: report.rebuilt_count,
|
||||
failed_count: report.failed_count,
|
||||
vacant_count: report.vacant_count,
|
||||
packer_ready_after_rebuild: packer_ready,
|
||||
staging_ready_after_rebuild: staging_ready,
|
||||
adapter_info: format!("{:?}", device.limits().max_texture_dimension_2d),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
pub mod audio_capture_to_mix;
|
||||
pub mod encoder_handoff_dryrun;
|
||||
pub mod ffi_airlock_negative;
|
||||
pub mod gpu_device_loss_recovery;
|
||||
pub mod screen_capture_to_pool;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
pub const SCHEMA_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ScenarioReport {
|
||||
pub schema: u32,
|
||||
pub scenario: &'static str,
|
||||
pub platform: &'static str,
|
||||
pub status: &'static str,
|
||||
pub measurements: serde_json::Value,
|
||||
pub assertions: Vec<String>,
|
||||
}
|
||||
|
||||
impl ScenarioReport {
|
||||
pub fn pass(
|
||||
scenario: &'static str,
|
||||
measurements: serde_json::Value,
|
||||
assertions: Vec<String>,
|
||||
) -> Self {
|
||||
let report = Self {
|
||||
schema: SCHEMA_VERSION,
|
||||
scenario,
|
||||
platform: current_platform(),
|
||||
status: "pass",
|
||||
measurements,
|
||||
assertions,
|
||||
};
|
||||
assert_eq!(report.status, "pass", "pass constructor status invariant");
|
||||
assert_eq!(report.schema, SCHEMA_VERSION, "schema version stable");
|
||||
report
|
||||
}
|
||||
|
||||
pub fn fail(
|
||||
scenario: &'static str,
|
||||
measurements: serde_json::Value,
|
||||
assertions: Vec<String>,
|
||||
) -> Self {
|
||||
let report = Self {
|
||||
schema: SCHEMA_VERSION,
|
||||
scenario,
|
||||
platform: current_platform(),
|
||||
status: "fail",
|
||||
measurements,
|
||||
assertions,
|
||||
};
|
||||
assert_eq!(report.status, "fail", "fail constructor status invariant");
|
||||
assert_eq!(report.schema, SCHEMA_VERSION, "schema version stable");
|
||||
report
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScenarioReport {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let serialised = serde_json::to_string(self).map_err(|_| std::fmt::Error)?;
|
||||
f.write_str(&serialised)
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn current_platform() -> &'static str {
|
||||
if cfg!(target_os = "macos") {
|
||||
"macos"
|
||||
} else if cfg!(target_os = "linux") {
|
||||
"linux"
|
||||
} else if cfg!(target_os = "windows") {
|
||||
"windows"
|
||||
} else {
|
||||
"unknown"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use fluxer_screen_frame_bus::frame_pool::{CpuFrameBuilder, FramePool};
|
||||
use serde_json::json;
|
||||
|
||||
use super::ScenarioReport;
|
||||
|
||||
const FRAME_WIDTH_PX: u32 = 1920;
|
||||
const FRAME_HEIGHT_PX: u32 = 1080;
|
||||
const FRAME_BYTES_PER_PIXEL: u32 = 4;
|
||||
const STEADY_STATE_FRAMES: u32 = 64;
|
||||
|
||||
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
|
||||
let bytes_per_slot = (FRAME_WIDTH_PX as usize)
|
||||
.saturating_mul(FRAME_HEIGHT_PX as usize)
|
||||
.saturating_mul(FRAME_BYTES_PER_PIXEL as usize);
|
||||
assert!(bytes_per_slot > 0, "bytes per slot positive");
|
||||
assert!(
|
||||
bytes_per_slot <= 1 << 28,
|
||||
"bytes per slot within sanity cap"
|
||||
);
|
||||
let pool = CpuFrameBuilder::build_pool(bytes_per_slot).map_err(|err| {
|
||||
ScenarioReport::fail(
|
||||
"screen_capture_to_pool",
|
||||
json!({"reason": format!("pool init failed: {err:?}")}),
|
||||
Vec::new(),
|
||||
)
|
||||
})?;
|
||||
let measurements = drive_steady_state(&pool);
|
||||
let mut assertions = Vec::new();
|
||||
if measurements.steady_state_skipped != 0 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"screen_capture_to_pool",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec![format!(
|
||||
"steady state should not skip; observed {} skips",
|
||||
measurements.steady_state_skipped
|
||||
)],
|
||||
));
|
||||
}
|
||||
assertions.push("FramePool steady state shows zero skips".to_string());
|
||||
if measurements.in_flight_after_release != 0 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"screen_capture_to_pool",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec![format!(
|
||||
"expected in-flight to drop to zero after release; observed {}",
|
||||
measurements.in_flight_after_release
|
||||
)],
|
||||
));
|
||||
}
|
||||
assertions.push("PooledFrame Drop returns slot to free list".to_string());
|
||||
if measurements.acquired_total_post_run < STEADY_STATE_FRAMES as u64 {
|
||||
return Err(ScenarioReport::fail(
|
||||
"screen_capture_to_pool",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
vec!["acquired_total below expected".to_string()],
|
||||
));
|
||||
}
|
||||
assertions.push("acquired_total matches steady-state frame count".to_string());
|
||||
Ok(ScenarioReport::pass(
|
||||
"screen_capture_to_pool",
|
||||
serde_json::to_value(&measurements).unwrap_or(json!({})),
|
||||
assertions,
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
struct Measurements {
|
||||
pool_capacity: usize,
|
||||
steady_state_frames: u32,
|
||||
steady_state_skipped: u64,
|
||||
acquired_total_post_run: u64,
|
||||
in_flight_after_release: u64,
|
||||
bytes_per_slot: usize,
|
||||
}
|
||||
|
||||
fn drive_steady_state(pool: &FramePool) -> Measurements {
|
||||
let capacity = pool.capacity();
|
||||
assert!(capacity > 0, "pool capacity positive");
|
||||
let baseline_skipped = pool.skipped_total();
|
||||
for _ in 0..STEADY_STATE_FRAMES {
|
||||
let pooled = pool
|
||||
.try_acquire()
|
||||
.expect("steady-state acquire must succeed since prior frame dropped");
|
||||
let _ = pooled.slot_index();
|
||||
}
|
||||
let in_flight = pool.currently_in_flight();
|
||||
let skipped = pool.skipped_total().saturating_sub(baseline_skipped);
|
||||
Measurements {
|
||||
pool_capacity: capacity,
|
||||
steady_state_frames: STEADY_STATE_FRAMES,
|
||||
steady_state_skipped: skipped,
|
||||
acquired_total_post_run: pool.acquired_total(),
|
||||
in_flight_after_release: in_flight,
|
||||
bytes_per_slot: ((FRAME_WIDTH_PX as usize)
|
||||
.saturating_mul(FRAME_HEIGHT_PX as usize)
|
||||
.saturating_mul(FRAME_BYTES_PER_PIXEL as usize)),
|
||||
}
|
||||
}
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
INTEGRATION_TRANSFER_LIB_LOADED=1
|
||||
|
||||
integration_transfer_b64_encode() {
|
||||
local src_path="$1"
|
||||
if [ ! -f "$src_path" ]; then
|
||||
printf 'integration_transfer: missing source %s\n' "$src_path" >&2
|
||||
return 64
|
||||
fi
|
||||
if command -v base64 >/dev/null 2>&1; then
|
||||
base64 < "$src_path" | tr -d '\n'
|
||||
else
|
||||
printf 'integration_transfer: base64 not available\n' >&2
|
||||
return 64
|
||||
fi
|
||||
}
|
||||
|
||||
integration_transfer_ssh_write_text() {
|
||||
local ssh_target="$1"
|
||||
local password="$2"
|
||||
local dest_path="$3"
|
||||
local src_path="$4"
|
||||
local encoded
|
||||
encoded="$(integration_transfer_b64_encode "$src_path")" || return $?
|
||||
local powershell_cmd
|
||||
powershell_cmd="\$bytes = [System.Convert]::FromBase64String('${encoded}'); [System.IO.File]::WriteAllBytes('${dest_path}', \$bytes)"
|
||||
if command -v sshpass >/dev/null 2>&1; then
|
||||
sshpass -p "$password" ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "powershell -NoProfile -Command \"${powershell_cmd}\""
|
||||
else
|
||||
ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "powershell -NoProfile -Command \"${powershell_cmd}\""
|
||||
fi
|
||||
}
|
||||
|
||||
integration_transfer_ssh_run() {
|
||||
local ssh_target="$1"
|
||||
local password="$2"
|
||||
local command_line="$3"
|
||||
if command -v sshpass >/dev/null 2>&1; then
|
||||
sshpass -p "$password" ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "$command_line"
|
||||
else
|
||||
ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "$command_line"
|
||||
fi
|
||||
}
|
||||
|
||||
integration_transfer_ssh_powershell() {
|
||||
local ssh_target="$1"
|
||||
local password="$2"
|
||||
local powershell_script="$3"
|
||||
integration_transfer_ssh_run "$ssh_target" "$password" "powershell -NoProfile -Command \"${powershell_script}\""
|
||||
}
|
||||
|
||||
integration_transfer_prlctl_exec() {
|
||||
local vm_name="$1"
|
||||
local command_line="$2"
|
||||
if ! command -v prlctl >/dev/null 2>&1; then
|
||||
printf 'integration_transfer: prlctl not available\n' >&2
|
||||
return 64
|
||||
fi
|
||||
prlctl exec "$vm_name" "$command_line"
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
*.json
|
||||
latest-summary.txt
|
||||
Executable
+183
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
PLATFORM=""
|
||||
ONLY_SCENARIO=""
|
||||
DRY_RUN="0"
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--platform=*) PLATFORM="${arg#--platform=}" ;;
|
||||
--only=*) ONLY_SCENARIO="${arg#--only=}" ;;
|
||||
--dry-run) DRY_RUN="1" ;;
|
||||
-h|--help)
|
||||
cat <<'EOF'
|
||||
usage: runner.sh --platform=<macos|linux-fedora|windows-tailnet|windows-vm> [--only=<scenario>] [--dry-run]
|
||||
|
||||
--platform=NAME Required. Selects the harness matrix entry.
|
||||
--only=NAME Optional. Run a single scenario (script basename without .sh).
|
||||
--dry-run Print which scenarios would run without invoking them.
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
printf 'runner.sh: unknown argument %s\n' "$arg" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$PLATFORM" ]; then
|
||||
printf 'runner.sh: --platform=<name> is required\n' >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RESULTS_DIR="${RUNNER_DIR}/results"
|
||||
SCENARIOS_DIR="${RUNNER_DIR}/scenarios"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
case "$PLATFORM" in
|
||||
macos)
|
||||
SCENARIO_LIST=(
|
||||
scenario_audio_capture_to_mix
|
||||
scenario_screen_capture_to_pool
|
||||
scenario_gpu_device_loss_recovery
|
||||
scenario_event_log_replay_determinism
|
||||
scenario_ffi_airlock_negative_space
|
||||
scenario_encoder_handoff_dryrun
|
||||
)
|
||||
;;
|
||||
linux-fedora)
|
||||
SCENARIO_LIST=(
|
||||
scenario_audio_capture_to_mix
|
||||
scenario_screen_capture_to_pool
|
||||
scenario_gpu_device_loss_recovery
|
||||
scenario_event_log_replay_determinism
|
||||
scenario_ffi_airlock_negative_space
|
||||
scenario_encoder_handoff_dryrun
|
||||
)
|
||||
;;
|
||||
windows-tailnet)
|
||||
SCENARIO_LIST=(
|
||||
scenario_audio_capture_to_mix
|
||||
scenario_screen_capture_to_pool
|
||||
scenario_gpu_device_loss_recovery
|
||||
scenario_event_log_replay_determinism
|
||||
scenario_ffi_airlock_negative_space
|
||||
scenario_encoder_handoff_dryrun
|
||||
)
|
||||
;;
|
||||
windows-vm)
|
||||
SCENARIO_LIST=(
|
||||
scenario_audio_capture_to_mix
|
||||
scenario_screen_capture_to_pool
|
||||
scenario_event_log_replay_determinism
|
||||
scenario_ffi_airlock_negative_space
|
||||
scenario_encoder_handoff_dryrun
|
||||
)
|
||||
;;
|
||||
*)
|
||||
printf 'runner.sh: unknown platform %s; expected macos|linux-fedora|windows-tailnet|windows-vm\n' "$PLATFORM" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -n "$ONLY_SCENARIO" ]; then
|
||||
FILTERED=()
|
||||
for s in "${SCENARIO_LIST[@]}"; do
|
||||
if [ "$s" = "$ONLY_SCENARIO" ]; then
|
||||
FILTERED+=("$s")
|
||||
fi
|
||||
done
|
||||
SCENARIO_LIST=("${FILTERED[@]}")
|
||||
if [ "${#SCENARIO_LIST[@]}" -eq 0 ]; then
|
||||
printf 'runner.sh: scenario %s not in platform matrix for %s\n' "$ONLY_SCENARIO" "$PLATFORM" >&2
|
||||
exit 64
|
||||
fi
|
||||
fi
|
||||
|
||||
TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
RESULT_FILE="${RESULTS_DIR}/${PLATFORM}-${TIMESTAMP}.json"
|
||||
SUMMARY_FILE="${RESULTS_DIR}/latest-summary.txt"
|
||||
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
printf 'Would run on platform=%s:\n' "$PLATFORM"
|
||||
for s in "${SCENARIO_LIST[@]}"; do
|
||||
printf ' %s\n' "$s"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
RESULT_ENTRIES=()
|
||||
SUMMARY_LINES=()
|
||||
|
||||
emit_result_entry() {
|
||||
local scenario="$1"
|
||||
local status="$2"
|
||||
local body="$3"
|
||||
local entry
|
||||
entry="{\"scenario\":\"${scenario}\",\"status\":\"${status}\",\"output\":${body}}"
|
||||
RESULT_ENTRIES+=("$entry")
|
||||
}
|
||||
|
||||
for scenario in "${SCENARIO_LIST[@]}"; do
|
||||
script="${SCENARIOS_DIR}/${scenario}.sh"
|
||||
if [ ! -x "$script" ]; then
|
||||
chmod +x "$script" 2>/dev/null || true
|
||||
fi
|
||||
if [ ! -f "$script" ]; then
|
||||
emit_result_entry "$scenario" "missing" "{\"reason\":\"scenario script not found at ${script}\"}"
|
||||
SUMMARY_LINES+=("[MISS] ${scenario}")
|
||||
FAILED=$((FAILED + 1))
|
||||
continue
|
||||
fi
|
||||
set +e
|
||||
raw_output="$("$script" 2>&1)"
|
||||
exit_code=$?
|
||||
set -e
|
||||
last_json_line="$(printf '%s\n' "$raw_output" | awk '/^\{.*\}$/ {last=$0} END {print last}')"
|
||||
if [ -z "$last_json_line" ]; then
|
||||
emit_result_entry "$scenario" "fail" "{\"reason\":\"scenario produced no JSON line\",\"raw\":$(jq -Rs . <<<"$raw_output" 2>/dev/null || echo '\"<unparseable>\"')}"
|
||||
SUMMARY_LINES+=("[FAIL] ${scenario} (no json)")
|
||||
FAILED=$((FAILED + 1))
|
||||
continue
|
||||
fi
|
||||
if [ "$exit_code" -eq 0 ]; then
|
||||
emit_result_entry "$scenario" "pass" "$last_json_line"
|
||||
SUMMARY_LINES+=("[PASS] ${scenario}")
|
||||
PASSED=$((PASSED + 1))
|
||||
else
|
||||
emit_result_entry "$scenario" "fail" "$last_json_line"
|
||||
SUMMARY_LINES+=("[FAIL] ${scenario} (exit ${exit_code})")
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
ENTRIES_JOINED="$(IFS=,; echo "${RESULT_ENTRIES[*]}")"
|
||||
printf '{"schema":1,"platform":"%s","timestamp":"%s","passed":%d,"failed":%d,"results":[%s]}\n' \
|
||||
"$PLATFORM" "$TIMESTAMP" "$PASSED" "$FAILED" "$ENTRIES_JOINED" > "$RESULT_FILE"
|
||||
|
||||
{
|
||||
printf 'platform: %s\n' "$PLATFORM"
|
||||
printf 'timestamp: %s\n' "$TIMESTAMP"
|
||||
printf 'passed: %d\n' "$PASSED"
|
||||
printf 'failed: %d\n' "$FAILED"
|
||||
printf 'scenarios:\n'
|
||||
for line in "${SUMMARY_LINES[@]}"; do
|
||||
printf ' %s\n' "$line"
|
||||
done
|
||||
printf 'result_file: %s\n' "$RESULT_FILE"
|
||||
} > "$SUMMARY_FILE"
|
||||
|
||||
cat "$SUMMARY_FILE"
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
if [ -z "${INTEGRATION_HARNESS_ROOT:-}" ]; then
|
||||
SCENARIO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
INTEGRATION_HARNESS_ROOT="$(cd "${SCENARIO_DIR}/.." && pwd)"
|
||||
fi
|
||||
export INTEGRATION_HARNESS_ROOT
|
||||
|
||||
INTEGRATION_REPO_ROOT="$(cd "${INTEGRATION_HARNESS_ROOT}/../.." && pwd)"
|
||||
export INTEGRATION_REPO_ROOT
|
||||
|
||||
detect_platform() {
|
||||
local uname_out
|
||||
uname_out="$(uname -s 2>/dev/null || echo unknown)"
|
||||
case "$uname_out" in
|
||||
Darwin) echo "macos" ;;
|
||||
Linux) echo "linux" ;;
|
||||
MINGW*|MSYS*|CYGWIN*|Windows_NT) echo "windows" ;;
|
||||
*)
|
||||
if [ -n "${OS:-}" ] && [ "$OS" = "Windows_NT" ]; then
|
||||
echo "windows"
|
||||
else
|
||||
echo "unknown"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
driver_target_dir() {
|
||||
local platform="$1"
|
||||
echo "${INTEGRATION_HARNESS_ROOT}/integration-driver/target-${platform}"
|
||||
}
|
||||
|
||||
driver_binary_path() {
|
||||
local platform="$1"
|
||||
local target_dir
|
||||
target_dir="$(driver_target_dir "$platform")"
|
||||
if [ "$platform" = "windows" ]; then
|
||||
echo "${target_dir}/debug/integration-driver.exe"
|
||||
else
|
||||
echo "${target_dir}/debug/integration-driver"
|
||||
fi
|
||||
}
|
||||
|
||||
driver_binary_runs() {
|
||||
local binary="$1"
|
||||
if [ ! -f "$binary" ]; then
|
||||
return 1
|
||||
fi
|
||||
if "$binary" --help >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if "$binary" 2>/dev/null | head -c 1 >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
ensure_driver_built() {
|
||||
local platform
|
||||
platform="$(detect_platform)"
|
||||
local binary
|
||||
binary="$(driver_binary_path "$platform")"
|
||||
local target_dir
|
||||
target_dir="$(driver_target_dir "$platform")"
|
||||
if ! driver_binary_runs "$binary"; then
|
||||
( cd "${INTEGRATION_HARNESS_ROOT}/integration-driver" && CARGO_TARGET_DIR="$target_dir" cargo build >&2 ) || return $?
|
||||
fi
|
||||
if [ ! -f "$binary" ]; then
|
||||
printf '{"schema":1,"scenario":"%s","platform":"%s","status":"fail","measurements":{"reason":"driver binary missing after build"},"assertions":[]}\n' \
|
||||
"${SCENARIO_NAME:-unknown}" "$platform"
|
||||
return 70
|
||||
fi
|
||||
echo "$binary"
|
||||
}
|
||||
|
||||
run_rust_subcommand() {
|
||||
local subcommand="$1"
|
||||
shift || true
|
||||
local binary
|
||||
binary="$(ensure_driver_built)" || return $?
|
||||
"$binary" "$subcommand" "$@"
|
||||
}
|
||||
|
||||
run_ts_driver() {
|
||||
local entry="$1"
|
||||
shift || true
|
||||
local resolved="${INTEGRATION_REPO_ROOT}/${entry}"
|
||||
if [ ! -f "$resolved" ]; then
|
||||
printf '{"schema":1,"scenario":"%s","platform":"%s","status":"fail","measurements":{"reason":"ts driver entry missing: %s"},"assertions":[]}\n' \
|
||||
"${SCENARIO_NAME:-unknown}" "$(detect_platform)" "$resolved"
|
||||
return 70
|
||||
fi
|
||||
local pnpm_command
|
||||
pnpm_command="$(resolve_pnpm_command)" || {
|
||||
printf '{"schema":1,"scenario":"%s","platform":"%s","status":"fail","measurements":{"reason":"pnpm not available"},"assertions":[]}\n' \
|
||||
"${SCENARIO_NAME:-unknown}" "$(detect_platform)"
|
||||
return 70
|
||||
}
|
||||
if should_use_dlx_tsx; then
|
||||
( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command dlx tsx "$entry" "$@" )
|
||||
elif ( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command exec tsx --version >/dev/null 2>&1 ); then
|
||||
( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command exec tsx "$entry" "$@" )
|
||||
else
|
||||
( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command dlx tsx "$entry" "$@" )
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_pnpm_command() {
|
||||
if command -v pnpm >/dev/null 2>&1; then
|
||||
echo "pnpm"
|
||||
return 0
|
||||
fi
|
||||
if command -v corepack >/dev/null 2>&1; then
|
||||
echo "corepack pnpm"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
should_use_dlx_tsx() {
|
||||
if [ "${FLUXER_INTEGRATION_TSX_MODE:-}" = "dlx" ]; then
|
||||
return 0
|
||||
fi
|
||||
case "$INTEGRATION_REPO_ROOT" in
|
||||
/media/psf/*|/mnt/psf/*)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
SCENARIO_NAME="scenario_audio_capture_to_mix"
|
||||
export SCENARIO_NAME
|
||||
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
|
||||
|
||||
run_rust_subcommand "audio-capture-to-mix"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
SCENARIO_NAME="scenario_encoder_handoff_dryrun"
|
||||
export SCENARIO_NAME
|
||||
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
|
||||
|
||||
run_rust_subcommand "encoder-handoff-dryrun"
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
SCENARIO_NAME="scenario_event_log_replay_determinism"
|
||||
export SCENARIO_NAME
|
||||
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
|
||||
|
||||
EXPECTED_FILE="${INTEGRATION_HARNESS_ROOT}/expected/event_log_replay_determinism.json"
|
||||
EXPECTED_HASH=""
|
||||
if [ -f "$EXPECTED_FILE" ]; then
|
||||
EXPECTED_HASH="$(grep -E '"snapshot_hash"' "$EXPECTED_FILE" | head -1 | sed -E 's/.*"snapshot_hash"[^"]*"([^"]+)".*/\1/' || true)"
|
||||
fi
|
||||
|
||||
if [ -n "$EXPECTED_HASH" ]; then
|
||||
run_ts_driver "tools/integration/ts-driver/event_log_replay_determinism.ts" "$EXPECTED_HASH"
|
||||
else
|
||||
run_ts_driver "tools/integration/ts-driver/event_log_replay_determinism.ts"
|
||||
fi
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
SCENARIO_NAME="scenario_ffi_airlock_negative_space"
|
||||
export SCENARIO_NAME
|
||||
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
|
||||
|
||||
run_rust_subcommand "ffi-airlock-negative" "invalid-sample-rate"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
SCENARIO_NAME="scenario_gpu_device_loss_recovery"
|
||||
export SCENARIO_NAME
|
||||
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
|
||||
|
||||
run_rust_subcommand "gpu-device-loss-recovery"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
SCENARIO_NAME="scenario_screen_capture_to_pool"
|
||||
export SCENARIO_NAME
|
||||
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
|
||||
|
||||
run_rust_subcommand "screen-capture-to-pool"
|
||||
@@ -0,0 +1,118 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {VoiceEngineV2Simulator} from '../../../packages/voice_engine_v2/src/simulation/Simulator';
|
||||
import {defineNetworkPartitionDuringScreenShareScenario} from '../../../packages/voice_engine_v2/src/simulation/scenarios/networkPartitionDuringScreenShare';
|
||||
|
||||
const REPORT_SCHEMA = 1;
|
||||
const SCENARIO_NAME = 'event_log_replay_determinism';
|
||||
const SEED = 4242;
|
||||
|
||||
interface Report {
|
||||
schema: number;
|
||||
scenario: string;
|
||||
platform: string;
|
||||
status: 'pass' | 'fail';
|
||||
measurements: Record<string, unknown>;
|
||||
assertions: Array<string>;
|
||||
}
|
||||
|
||||
function platform(): string {
|
||||
if (process.platform === 'darwin') return 'macos';
|
||||
if (process.platform === 'linux') return 'linux';
|
||||
if (process.platform === 'win32') return 'windows';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
async function runOnce(): Promise<{snapshotHash: string; eventCount: number; finalTick: number}> {
|
||||
const scenario = defineNetworkPartitionDuringScreenShareScenario(SEED);
|
||||
assert.ok(scenario.workload, 'scenario must yield a workload');
|
||||
const simulator = new VoiceEngineV2Simulator({
|
||||
seed: SEED,
|
||||
workload: scenario.workload,
|
||||
faults: scenario.faultPlan,
|
||||
mode: scenario.mode,
|
||||
});
|
||||
const result = await simulator.run();
|
||||
assert.ok(typeof result.snapshotHash === 'string', 'snapshot hash must be a string');
|
||||
assert.equal(result.snapshotHash.length, 8, 'snapshot hash is eight hex characters');
|
||||
return {
|
||||
snapshotHash: result.snapshotHash,
|
||||
eventCount: result.eventLog.length,
|
||||
finalTick: result.finalTick,
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const expectedHashArg = process.argv[2] ?? null;
|
||||
const first = await runOnce();
|
||||
const second = await runOnce();
|
||||
const assertions: Array<string> = [];
|
||||
if (first.snapshotHash !== second.snapshotHash) {
|
||||
emit({
|
||||
schema: REPORT_SCHEMA,
|
||||
scenario: SCENARIO_NAME,
|
||||
platform: platform(),
|
||||
status: 'fail',
|
||||
measurements: {first, second},
|
||||
assertions: ['two runs of the same scenario must agree on snapshot hash'],
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
assertions.push('two runs of the same scenario agree on snapshot hash');
|
||||
if (first.eventCount !== second.eventCount) {
|
||||
emit({
|
||||
schema: REPORT_SCHEMA,
|
||||
scenario: SCENARIO_NAME,
|
||||
platform: platform(),
|
||||
status: 'fail',
|
||||
measurements: {first, second},
|
||||
assertions: ['event log length must agree across runs'],
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
assertions.push('event log length agrees across runs');
|
||||
if (expectedHashArg !== null && expectedHashArg !== first.snapshotHash) {
|
||||
emit({
|
||||
schema: REPORT_SCHEMA,
|
||||
scenario: SCENARIO_NAME,
|
||||
platform: platform(),
|
||||
status: 'fail',
|
||||
measurements: {expected: expectedHashArg, observed: first.snapshotHash, runs: [first, second]},
|
||||
assertions: [`snapshot hash mismatch: expected ${expectedHashArg}, observed ${first.snapshotHash}`],
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
if (expectedHashArg !== null) {
|
||||
assertions.push('snapshot hash matches expected fixture');
|
||||
}
|
||||
emit({
|
||||
schema: REPORT_SCHEMA,
|
||||
scenario: SCENARIO_NAME,
|
||||
platform: platform(),
|
||||
status: 'pass',
|
||||
measurements: {
|
||||
snapshot_hash: first.snapshotHash,
|
||||
event_count: first.eventCount,
|
||||
final_tick: first.finalTick,
|
||||
seed: SEED,
|
||||
},
|
||||
assertions,
|
||||
});
|
||||
}
|
||||
|
||||
function emit(report: Report): void {
|
||||
process.stdout.write(`${JSON.stringify(report)}\n`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
emit({
|
||||
schema: REPORT_SCHEMA,
|
||||
scenario: SCENARIO_NAME,
|
||||
platform: platform(),
|
||||
status: 'fail',
|
||||
measurements: {error: String(error)},
|
||||
assertions: ['driver threw before finishing scenario'],
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "fluxer-marketing-update-gettext-catalogs"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.102"
|
||||
chrono = "0.4.45"
|
||||
serde_json = "1.0.150"
|
||||
syn = { version = "2.0.117", features = ["full", "visit"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.27.0"
|
||||
@@ -0,0 +1,660 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use chrono::{Local, NaiveDate};
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use syn::{
|
||||
Expr, ExprLit, ExprMacro, Ident, Lit, Macro, Token, braced,
|
||||
parse::{Parse, ParseStream},
|
||||
visit::{self, Visit},
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct Descriptor {
|
||||
const_name: String,
|
||||
key: String,
|
||||
message: String,
|
||||
comment: String,
|
||||
}
|
||||
|
||||
pub fn find_marketing_root(current_dir: &Path) -> Result<PathBuf> {
|
||||
for ancestor in current_dir.ancestors() {
|
||||
if is_marketing_root(ancestor) {
|
||||
return Ok(ancestor.to_path_buf());
|
||||
}
|
||||
let child = ancestor.join("fluxer_marketing");
|
||||
if is_marketing_root(&child) {
|
||||
return Ok(child);
|
||||
}
|
||||
}
|
||||
bail!(
|
||||
"could not find fluxer_marketing from {}; run from the repository root or pass fluxer_marketing explicitly",
|
||||
current_dir.display()
|
||||
);
|
||||
}
|
||||
|
||||
pub fn update_catalogs(root: &Path) -> Result<()> {
|
||||
update_catalogs_with_date(root, Local::now().date_naive())
|
||||
}
|
||||
|
||||
pub fn update_catalogs_with_date(root: &Path, today: NaiveDate) -> Result<()> {
|
||||
let descriptors = parse_descriptors(root)?;
|
||||
let locales_dir = root.join("locales");
|
||||
if !locales_dir.is_dir() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut locales = fs::read_dir(&locales_dir)
|
||||
.with_context(|| format!("failed to read {}", locales_dir.display()))?
|
||||
.map(|entry| entry.map(|entry| entry.path()))
|
||||
.collect::<std::io::Result<Vec<_>>>()
|
||||
.with_context(|| format!("failed to read {}", locales_dir.display()))?
|
||||
.into_iter()
|
||||
.filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("po"))
|
||||
.filter_map(|path| {
|
||||
path.file_stem()
|
||||
.and_then(|stem| stem.to_str())
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
locales.sort();
|
||||
|
||||
for locale in locales {
|
||||
write_catalog(&locales_dir, &locale, &descriptors, today)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_marketing_root(path: &Path) -> bool {
|
||||
path.join("src/i18n/descriptors.rs").is_file() && path.join("locales").is_dir()
|
||||
}
|
||||
|
||||
fn parse_descriptors(root: &Path) -> Result<Vec<Descriptor>> {
|
||||
let descriptors_path = root.join("src/i18n/descriptors.rs");
|
||||
let descriptors_dir = root.join("src/i18n/descriptors");
|
||||
let mut descriptors = Vec::new();
|
||||
|
||||
for path in descriptor_sources(&descriptors_path, &descriptors_dir)? {
|
||||
let source = fs::read_to_string(&path)
|
||||
.with_context(|| format!("failed to read {}", path.display()))?;
|
||||
let file = syn::parse_file(&source)
|
||||
.with_context(|| format!("failed to parse {}", path.display()))?;
|
||||
let mut visitor = MarketingMessageVisitor::new(&path);
|
||||
visitor.visit_file(&file);
|
||||
for descriptor in visitor.descriptors {
|
||||
descriptors.push(descriptor?);
|
||||
}
|
||||
}
|
||||
|
||||
if descriptors.is_empty() {
|
||||
bail!(
|
||||
"no descriptors found under {} / {}",
|
||||
descriptors_path.display(),
|
||||
descriptors_dir.display()
|
||||
);
|
||||
}
|
||||
Ok(descriptors)
|
||||
}
|
||||
|
||||
fn descriptor_sources(descriptors_path: &Path, descriptors_dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
let mut sources = vec![descriptors_path.to_path_buf()];
|
||||
if descriptors_dir.is_dir() {
|
||||
let mut children = fs::read_dir(descriptors_dir)
|
||||
.with_context(|| format!("failed to read {}", descriptors_dir.display()))?
|
||||
.map(|entry| entry.map(|entry| entry.path()))
|
||||
.collect::<std::io::Result<Vec<_>>>()
|
||||
.with_context(|| format!("failed to read {}", descriptors_dir.display()))?
|
||||
.into_iter()
|
||||
.filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("rs"))
|
||||
.collect::<Vec<_>>();
|
||||
children.sort();
|
||||
sources.extend(children);
|
||||
}
|
||||
Ok(sources)
|
||||
}
|
||||
|
||||
struct MarketingMessageVisitor<'a> {
|
||||
descriptor_file: &'a Path,
|
||||
descriptors: Vec<Result<Descriptor>>,
|
||||
}
|
||||
|
||||
impl<'a> MarketingMessageVisitor<'a> {
|
||||
fn new(descriptor_file: &'a Path) -> Self {
|
||||
Self {
|
||||
descriptor_file,
|
||||
descriptors: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'ast> Visit<'ast> for MarketingMessageVisitor<'_> {
|
||||
fn visit_macro(&mut self, node: &'ast Macro) {
|
||||
if is_marketing_message_macro(node) {
|
||||
let descriptor = node
|
||||
.parse_body::<MarketingMessageInput>()
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"failed to parse marketing_message! descriptor in {}",
|
||||
self.descriptor_file.display()
|
||||
)
|
||||
})
|
||||
.and_then(|input| input.into_descriptor(self.descriptor_file));
|
||||
self.descriptors.push(descriptor);
|
||||
}
|
||||
visit::visit_macro(self, node);
|
||||
}
|
||||
}
|
||||
|
||||
struct MarketingMessageInput {
|
||||
const_name: Ident,
|
||||
key: Expr,
|
||||
message: Expr,
|
||||
comment: Expr,
|
||||
}
|
||||
|
||||
impl MarketingMessageInput {
|
||||
fn into_descriptor(self, descriptor_file: &Path) -> Result<Descriptor> {
|
||||
let const_name = self.const_name.to_string();
|
||||
let key = expect_string_literal(&self.key, &const_name, "key", descriptor_file)?;
|
||||
let message = parse_descriptor_message(descriptor_file, &const_name, &self.message)?;
|
||||
let comment =
|
||||
expect_string_literal(&self.comment, &const_name, "comment", descriptor_file)?;
|
||||
Ok(Descriptor {
|
||||
const_name,
|
||||
key,
|
||||
message,
|
||||
comment,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Parse for MarketingMessageInput {
|
||||
fn parse(input: ParseStream) -> syn::Result<Self> {
|
||||
input.parse::<Token![pub]>()?;
|
||||
input.parse::<Token![const]>()?;
|
||||
let const_name = input.parse()?;
|
||||
input.parse::<Token![=]>()?;
|
||||
|
||||
let body;
|
||||
braced!(body in input);
|
||||
let key = parse_expected_field(&body, "key")?;
|
||||
body.parse::<Token![,]>()?;
|
||||
let message = parse_expected_field(&body, "message")?;
|
||||
body.parse::<Token![,]>()?;
|
||||
let comment = parse_expected_field(&body, "comment")?;
|
||||
if body.peek(Token![,]) {
|
||||
body.parse::<Token![,]>()?;
|
||||
}
|
||||
if !body.is_empty() {
|
||||
return Err(body.error("unexpected descriptor field"));
|
||||
}
|
||||
|
||||
input.parse::<Token![;]>()?;
|
||||
if !input.is_empty() {
|
||||
return Err(input.error("unexpected tokens after descriptor"));
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
const_name,
|
||||
key,
|
||||
message,
|
||||
comment,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_expected_field(input: ParseStream, expected_name: &str) -> syn::Result<Expr> {
|
||||
let name: Ident = input.parse()?;
|
||||
if name != expected_name {
|
||||
return Err(syn::Error::new(
|
||||
name.span(),
|
||||
format!("expected `{expected_name}` field"),
|
||||
));
|
||||
}
|
||||
input.parse::<Token![:]>()?;
|
||||
input.parse()
|
||||
}
|
||||
|
||||
fn is_marketing_message_macro(node: &Macro) -> bool {
|
||||
if node.path.leading_colon.is_some() {
|
||||
return false;
|
||||
}
|
||||
let segments = node
|
||||
.path
|
||||
.segments
|
||||
.iter()
|
||||
.map(|segment| segment.ident.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
matches!(segments.as_slice(), [name] if name == "marketing_message")
|
||||
|| matches!(segments.as_slice(), [root, name] if root == "crate" && name == "marketing_message")
|
||||
}
|
||||
|
||||
fn expect_string_literal(
|
||||
expr: &Expr,
|
||||
const_name: &str,
|
||||
field_name: &str,
|
||||
descriptor_file: &Path,
|
||||
) -> Result<String> {
|
||||
if let Expr::Lit(ExprLit {
|
||||
lit: Lit::Str(value),
|
||||
..
|
||||
}) = expr
|
||||
{
|
||||
return Ok(value.value());
|
||||
}
|
||||
bail!(
|
||||
"descriptor {} field {} must be a string literal in {}",
|
||||
const_name,
|
||||
field_name,
|
||||
descriptor_file.display()
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_descriptor_message(
|
||||
descriptor_file: &Path,
|
||||
const_name: &str,
|
||||
message: &Expr,
|
||||
) -> Result<String> {
|
||||
if let Expr::Lit(ExprLit {
|
||||
lit: Lit::Str(value),
|
||||
..
|
||||
}) = message
|
||||
{
|
||||
return Ok(value.value());
|
||||
}
|
||||
let Expr::Macro(ExprMacro { mac, .. }) = message else {
|
||||
bail!(
|
||||
"descriptor {} message must be a string literal or include_str!(...) in {}",
|
||||
const_name,
|
||||
descriptor_file.display()
|
||||
);
|
||||
};
|
||||
if !is_include_str_macro(mac) {
|
||||
bail!(
|
||||
"descriptor {} message must be a string literal or include_str!(...) in {}",
|
||||
const_name,
|
||||
descriptor_file.display()
|
||||
);
|
||||
}
|
||||
let relative_path = mac.parse_body::<syn::LitStr>().with_context(|| {
|
||||
format!(
|
||||
"invalid include_str! descriptor message in {} for {}",
|
||||
descriptor_file.display(),
|
||||
const_name
|
||||
)
|
||||
})?;
|
||||
let source_path = descriptor_file
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow!("descriptor file should have a parent directory"))?
|
||||
.join(relative_path.value());
|
||||
fs::read_to_string(&source_path)
|
||||
.with_context(|| format!("failed to read descriptor source {}", source_path.display()))
|
||||
}
|
||||
|
||||
fn is_include_str_macro(node: &Macro) -> bool {
|
||||
node.path.leading_colon.is_none()
|
||||
&& node.path.segments.len() == 1
|
||||
&& node.path.segments[0].ident == "include_str"
|
||||
}
|
||||
|
||||
fn parse_existing_translations(path: &Path) -> Result<BTreeMap<String, String>> {
|
||||
if !path.exists() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
let text =
|
||||
fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?;
|
||||
let mut translations = BTreeMap::new();
|
||||
for entry in split_po_entries(&text) {
|
||||
let context = parse_po_field(entry, "msgctxt")?;
|
||||
let msgstr = parse_po_field(entry, "msgstr")?;
|
||||
if let (Some(context), Some(msgstr)) = (context, msgstr) {
|
||||
translations.insert(context, msgstr);
|
||||
}
|
||||
}
|
||||
Ok(translations)
|
||||
}
|
||||
|
||||
fn split_po_entries(text: &str) -> Vec<&str> {
|
||||
let lines = text.split_inclusive('\n').collect::<Vec<_>>();
|
||||
let mut entries = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut offset = 0;
|
||||
for line in lines {
|
||||
let line_start = offset;
|
||||
offset += line.len();
|
||||
if line.trim().is_empty() {
|
||||
entries.push(&text[start..line_start]);
|
||||
start = offset;
|
||||
}
|
||||
}
|
||||
if start <= text.len() {
|
||||
entries.push(&text[start..]);
|
||||
}
|
||||
entries
|
||||
}
|
||||
|
||||
fn parse_po_field(entry: &str, field: &str) -> Result<Option<String>> {
|
||||
let prefix = format!("{field} ");
|
||||
let lines = entry.lines().collect::<Vec<_>>();
|
||||
for (index, line) in lines.iter().enumerate() {
|
||||
let Some(value) = line.strip_prefix(&prefix) else {
|
||||
continue;
|
||||
};
|
||||
let mut values = vec![value];
|
||||
for continuation in lines.iter().skip(index + 1) {
|
||||
if !continuation.starts_with('"') {
|
||||
break;
|
||||
}
|
||||
values.push(continuation);
|
||||
}
|
||||
|
||||
let mut parsed = String::new();
|
||||
for value in values {
|
||||
parsed.push_str(&unescape_po_string(value)?);
|
||||
}
|
||||
return Ok(Some(parsed));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn unescape_po_string(quoted: &str) -> Result<String> {
|
||||
if let Ok(value) = serde_json::from_str::<String>(quoted) {
|
||||
return Ok(value);
|
||||
}
|
||||
|
||||
let stripped = quoted.trim();
|
||||
if !stripped.starts_with('"') || !stripped.ends_with('"') {
|
||||
bail!("not a PO quoted string: {quoted:?}");
|
||||
}
|
||||
let inner = &stripped[1..stripped.len() - 1];
|
||||
let mut result = String::new();
|
||||
let mut chars = inner.chars();
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch == '\\' {
|
||||
if let Some(next) = chars.next() {
|
||||
match next {
|
||||
'n' => result.push('\n'),
|
||||
't' => result.push('\t'),
|
||||
'\\' => result.push('\\'),
|
||||
'"' => result.push('"'),
|
||||
other => {
|
||||
result.push(ch);
|
||||
result.push(other);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
result.push(ch);
|
||||
}
|
||||
} else {
|
||||
result.push(ch);
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn write_catalog(
|
||||
locales_dir: &Path,
|
||||
locale: &str,
|
||||
descriptors: &[Descriptor],
|
||||
today: NaiveDate,
|
||||
) -> Result<()> {
|
||||
let path = locales_dir.join(format!("{locale}.po"));
|
||||
let existing = parse_existing_translations(&path)?;
|
||||
let today = today.to_string();
|
||||
let mut lines = vec![
|
||||
"# SPDX-License-Identifier: AGPL-3.0-or-later".to_owned(),
|
||||
"#".to_owned(),
|
||||
format!("# Fluxer marketing gettext catalog for {locale}."),
|
||||
"msgid \"\"".to_owned(),
|
||||
"msgstr \"\"".to_owned(),
|
||||
"\"Project-Id-Version: fluxer-marketing\\n\"".to_owned(),
|
||||
format!("\"POT-Creation-Date: {today} 00:00+0000\\n\""),
|
||||
format!("\"PO-Revision-Date: {today} 00:00+0000\\n\""),
|
||||
format!("\"Language: {locale}\\n\""),
|
||||
"\"MIME-Version: 1.0\\n\"".to_owned(),
|
||||
"\"Content-Type: text/plain; charset=UTF-8\\n\"".to_owned(),
|
||||
"\"Content-Transfer-Encoding: 8bit\\n\"".to_owned(),
|
||||
"\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"".to_owned(),
|
||||
String::new(),
|
||||
];
|
||||
|
||||
for descriptor in descriptors {
|
||||
let translated = if locale == "en-US" {
|
||||
descriptor.message.as_str()
|
||||
} else {
|
||||
existing
|
||||
.get(&descriptor.key)
|
||||
.map(String::as_str)
|
||||
.unwrap_or(&descriptor.message)
|
||||
};
|
||||
lines.push(format!("#. {}", descriptor.comment));
|
||||
lines.push(format!("#: fluxer_marketing/generated:{}", descriptor.key));
|
||||
write_field(&mut lines, "msgctxt", &descriptor.key);
|
||||
write_field(&mut lines, "msgid", &descriptor.message);
|
||||
write_field(&mut lines, "msgstr", translated);
|
||||
lines.push(String::new());
|
||||
}
|
||||
|
||||
fs::write(&path, lines.join("\n"))
|
||||
.with_context(|| format!("failed to write {}", path.display()))
|
||||
}
|
||||
|
||||
fn po_escape(value: &str) -> String {
|
||||
value
|
||||
.replace('\\', "\\\\")
|
||||
.replace('"', "\\\"")
|
||||
.replace('\n', "\\n")
|
||||
}
|
||||
|
||||
fn write_field(lines: &mut Vec<String>, name: &str, value: &str) {
|
||||
if !value.contains('\n') && value.chars().count() <= 90 {
|
||||
lines.push(format!(r#"{name} "{}""#, po_escape(value)));
|
||||
return;
|
||||
}
|
||||
|
||||
lines.push(format!(r#"{name} """#));
|
||||
for part in split_lines_keepends(value) {
|
||||
lines.push(format!(r#""{}""#, po_escape(part)));
|
||||
}
|
||||
}
|
||||
|
||||
fn split_lines_keepends(value: &str) -> Vec<&str> {
|
||||
let mut parts = Vec::new();
|
||||
let mut start = 0;
|
||||
for (index, ch) in value.char_indices() {
|
||||
if ch == '\n' {
|
||||
let end = index + ch.len_utf8();
|
||||
parts.push(&value[start..end]);
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
if start < value.len() {
|
||||
parts.push(&value[start..]);
|
||||
}
|
||||
if parts.is_empty() {
|
||||
parts.push("");
|
||||
}
|
||||
parts
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::NaiveDate;
|
||||
use std::fs;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn updates_catalogs_from_literals_and_include_str_sources() {
|
||||
let dir = tempdir().unwrap();
|
||||
let root = dir.path().join("fluxer_marketing");
|
||||
fs::create_dir_all(root.join("src/i18n/descriptors")).unwrap();
|
||||
fs::create_dir_all(root.join("locales")).unwrap();
|
||||
fs::write(
|
||||
root.join("src/i18n/descriptors.rs"),
|
||||
r#"
|
||||
marketing_message!(
|
||||
pub const SHORT_DESCRIPTOR = {
|
||||
key: "app.short",
|
||||
message: "Hello \"Fluxer\"",
|
||||
comment: "Shown on the fixture page with escaped quotes for translators.",
|
||||
};
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("src/i18n/descriptors/extra.rs"),
|
||||
r#"
|
||||
crate::marketing_message!(
|
||||
pub const BODY_DESCRIPTOR = {
|
||||
key: "content.body",
|
||||
message: include_str!("body.txt"),
|
||||
comment: "Body copy loaded from a markdown fixture and shown on a content page.",
|
||||
};
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("src/i18n/descriptors/body.txt"),
|
||||
"Body first line\nBody second \"quote\" \\ path\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("locales/en-US.po"),
|
||||
r#"
|
||||
msgctxt "app.short"
|
||||
msgid "Old"
|
||||
msgstr "Outdated"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("locales/sv-SE.po"),
|
||||
r#"
|
||||
msgctxt "app.short"
|
||||
msgid "Old"
|
||||
msgstr "Preserved short"
|
||||
|
||||
msgctxt "content.body"
|
||||
msgid "Old body"
|
||||
msgstr ""
|
||||
"Preserved line one\n"
|
||||
"Preserved \"quote\" \\ path\n"
|
||||
|
||||
msgctxt "obsolete"
|
||||
msgid "Obsolete"
|
||||
msgstr "Should disappear"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
update_catalogs_with_date(
|
||||
&root,
|
||||
NaiveDate::from_ymd_opt(2026, 6, 4).expect("valid date"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let en_us_path = root.join("locales/en-US.po");
|
||||
let sv_se_path = root.join("locales/sv-SE.po");
|
||||
let en_us_text = fs::read_to_string(&en_us_path).unwrap();
|
||||
let sv_se_text = fs::read_to_string(&sv_se_path).unwrap();
|
||||
|
||||
assert!(en_us_text.contains("\"POT-Creation-Date: 2026-06-04 00:00+0000\\n\""));
|
||||
assert!(en_us_text.contains("msgstr \"Hello \\\"Fluxer\\\"\""));
|
||||
assert!(en_us_text.contains(
|
||||
r#"msgid ""
|
||||
"Body first line\n"
|
||||
"Body second \"quote\" \\ path\n""#
|
||||
));
|
||||
assert!(sv_se_text.contains("msgstr \"Preserved short\""));
|
||||
assert!(!sv_se_text.contains("obsolete"));
|
||||
|
||||
let en_us = parse_existing_translations(&en_us_path).unwrap();
|
||||
let sv_se = parse_existing_translations(&sv_se_path).unwrap();
|
||||
assert_eq!(en_us["app.short"], "Hello \"Fluxer\"");
|
||||
assert_eq!(
|
||||
en_us["content.body"],
|
||||
"Body first line\nBody second \"quote\" \\ path\n"
|
||||
);
|
||||
assert_eq!(sv_se["app.short"], "Preserved short");
|
||||
assert_eq!(
|
||||
sv_se["content.body"],
|
||||
"Preserved line one\nPreserved \"quote\" \\ path\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_long_single_line_fields_as_multiline_po_strings() {
|
||||
let value = "x".repeat(91);
|
||||
let mut lines = Vec::new();
|
||||
|
||||
write_field(&mut lines, "msgid", &value);
|
||||
|
||||
assert_eq!(lines, vec!["msgid \"\"".to_owned(), format!("\"{value}\"")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unescapes_unknown_po_escapes_losslessly() {
|
||||
assert_eq!(unescape_po_string(r#""a\qb""#).unwrap(), r#"a\qb"#);
|
||||
assert_eq!(
|
||||
unescape_po_string(r#""line\nquote\"slash\\""#).unwrap(),
|
||||
"line\nquote\"slash\\"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_missing_include_str_source() {
|
||||
let dir = tempdir().unwrap();
|
||||
let root = dir.path().join("fluxer_marketing");
|
||||
fs::create_dir_all(root.join("src/i18n/descriptors")).unwrap();
|
||||
fs::create_dir_all(root.join("locales")).unwrap();
|
||||
fs::write(root.join("locales/en-US.po"), "").unwrap();
|
||||
fs::write(root.join("src/i18n/descriptors.rs"), "").unwrap();
|
||||
fs::write(
|
||||
root.join("src/i18n/descriptors/missing.rs"),
|
||||
r#"
|
||||
crate::marketing_message!(
|
||||
pub const MISSING_DESCRIPTOR = {
|
||||
key: "missing.body",
|
||||
message: include_str!("missing.txt"),
|
||||
comment: "Body copy loaded from a missing fixture file for failure handling.",
|
||||
};
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let err = update_catalogs_with_date(
|
||||
&root,
|
||||
NaiveDate::from_ymd_opt(2026, 6, 4).expect("valid date"),
|
||||
)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
|
||||
assert!(err.contains("failed to read descriptor source"));
|
||||
assert!(err.contains("missing.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_marketing_root_from_repo_or_package_directory() {
|
||||
let dir = tempdir().unwrap();
|
||||
let repo = dir.path();
|
||||
let root = repo.join("fluxer_marketing");
|
||||
fs::create_dir_all(root.join("src/i18n")).unwrap();
|
||||
fs::create_dir_all(root.join("locales")).unwrap();
|
||||
fs::write(root.join("src/i18n/descriptors.rs"), "").unwrap();
|
||||
let nested_tool_dir = repo.join("tools/marketing/update-gettext-catalogs");
|
||||
fs::create_dir_all(&nested_tool_dir).unwrap();
|
||||
|
||||
assert_eq!(find_marketing_root(repo).unwrap(), root);
|
||||
assert_eq!(
|
||||
find_marketing_root(&repo.join("fluxer_marketing")).unwrap(),
|
||||
root
|
||||
);
|
||||
assert_eq!(find_marketing_root(&nested_tool_dir).unwrap(), root);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::Result;
|
||||
use fluxer_marketing_update_gettext_catalogs::{find_marketing_root, update_catalogs};
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use std::process;
|
||||
|
||||
fn main() {
|
||||
if let Err(err) = run() {
|
||||
eprintln!("error: {err:#}");
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<()> {
|
||||
let mut args = env::args_os();
|
||||
let program = args
|
||||
.next()
|
||||
.and_then(|value| PathBuf::from(value).file_name().map(|name| name.to_owned()))
|
||||
.and_then(|name| name.into_string().ok())
|
||||
.unwrap_or_else(|| "fluxer-marketing-update-gettext-catalogs".to_owned());
|
||||
|
||||
let root = match (args.next(), args.next()) {
|
||||
(None, None) => find_marketing_root(&env::current_dir()?)?,
|
||||
(Some(root), None) => PathBuf::from(root),
|
||||
_ => {
|
||||
eprintln!("usage: {program} [marketing_root]");
|
||||
process::exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
update_catalogs(&root)
|
||||
}
|
||||
Reference in New Issue
Block a user