workspace: reconcile latest Fluxer into the open source release and continue development there

This commit is contained in:
Hampus Kraft
2026-06-16 06:41:37 +02:00
parent ee1f27fe1a
commit fa3e852575
19032 changed files with 10716201 additions and 1865276 deletions
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Parse criterion output and compare against a baseline.json.
Reads:
argv[1] path to the captured `cargo bench` log
BASELINE_PATH (env) path to baseline.json
BUDGET_PERCENT (env) allowed slowdown in percent (e.g. 5 means +5%)
Exits 0 if every bench mean is within +BUDGET of baseline median; 1 otherwise.
Unknown benches in the log are reported but do not fail the run.
Benches in the baseline that did not appear in the log are reported and fail.
"""
from __future__ import annotations
import json
import os
import re
import sys
from pathlib import Path
UNIT_NS = {
"ns": 1.0,
"us": 1_000.0,
"µs": 1_000.0,
"ms": 1_000_000.0,
"s": 1_000_000_000.0,
}
LINE_WITH_NAME_RE = re.compile(
r"^(?P<name>\S[^\t]*?)\s+time:\s*"
r"\[(?P<low>[\d.]+)\s*(?P<low_u>\S+)\s+"
r"(?P<mean>[\d.]+)\s*(?P<mean_u>\S+)\s+"
r"(?P<high>[\d.]+)\s*(?P<high_u>\S+)\]"
)
TIME_ONLY_RE = re.compile(
r"^\s*time:\s*"
r"\[(?P<low>[\d.]+)\s*(?P<low_u>\S+)\s+"
r"(?P<mean>[\d.]+)\s*(?P<mean_u>\S+)\s+"
r"(?P<high>[\d.]+)\s*(?P<high_u>\S+)\]"
)
def to_ns(value: float, unit: str) -> float:
if unit not in UNIT_NS:
raise ValueError(f"unknown criterion time unit: {unit!r}")
return value * UNIT_NS[unit]
def parse_log(log_path: Path) -> dict[str, dict[str, float]]:
measurements: dict[str, dict[str, float]] = {}
name_buffer: str | None = None
for raw in log_path.read_text(errors="replace").splitlines():
line = raw.rstrip()
if not line:
continue
if "change:" in line:
continue
if "time:" in line:
match = LINE_WITH_NAME_RE.match(line)
if match:
measurements[match.group("name").strip()] = build_measurement(match)
name_buffer = None
continue
match = TIME_ONLY_RE.match(line)
if match and name_buffer is not None:
measurements[name_buffer] = build_measurement(match)
name_buffer = None
continue
if line.startswith("Benchmarking "):
name = line[len("Benchmarking "):].strip()
if name.endswith(": Analyzing"):
name_buffer = name[: -len(": Analyzing")]
elif ":" not in name:
name_buffer = name
continue
return measurements
def build_measurement(match: re.Match[str]) -> dict[str, float]:
return {
"low_ns": to_ns(float(match.group("low")), match.group("low_u")),
"mean_ns": to_ns(float(match.group("mean")), match.group("mean_u")),
"high_ns": to_ns(float(match.group("high")), match.group("high_u")),
}
def compare(
baseline: dict,
measurements: dict[str, dict[str, float]],
budget_percent: float,
) -> tuple[list[str], list[str], list[str]]:
benches = baseline.get("benches", {})
if not isinstance(benches, dict):
raise ValueError("baseline.json missing 'benches' object")
regressions: list[str] = []
passes: list[str] = []
missing: list[str] = []
for name, entry in benches.items():
if not isinstance(entry, dict):
continue
if "median_ns" not in entry:
continue
baseline_ns = float(entry["median_ns"])
measured = measurements.get(name)
if measured is None:
missing.append(name)
continue
observed_ns = measured["mean_ns"]
delta = observed_ns - baseline_ns
delta_pct = (delta / baseline_ns) * 100.0 if baseline_ns > 0 else 0.0
effective_budget = float(entry.get("budget_percent_override", budget_percent))
budget_ns = baseline_ns * (effective_budget / 100.0)
status = "OK" if delta <= budget_ns else "REGRESSION"
summary = (
f" {status:11s} {name}: baseline={baseline_ns:.3f}ns "
f"observed={observed_ns:.3f}ns delta={delta_pct:+.2f}% "
f"(budget=+{effective_budget:.1f}%)"
)
if status == "OK":
passes.append(summary)
else:
regressions.append(summary)
return passes, regressions, missing
def main() -> int:
if len(sys.argv) != 2:
print("usage: _compare_criterion.py <bench-log>", file=sys.stderr)
return 2
log_path = Path(sys.argv[1])
baseline_path = Path(os.environ["BASELINE_PATH"])
budget = float(os.environ.get("BUDGET_PERCENT", "5"))
strict_missing = os.environ.get("STRICT_MISSING", "0") == "1"
baseline = json.loads(baseline_path.read_text())
measurements = parse_log(log_path)
if not measurements:
print("ERROR: no criterion measurements parsed from log", file=sys.stderr)
return 2
passes, regressions, missing = compare(baseline, measurements, budget)
for line in passes:
print(line)
for line in regressions:
print(line)
if missing and strict_missing:
print("MISSING in run (baseline expected but no measurement found):")
for name in missing:
print(f" {name}")
elif missing:
print(
f"(info: {len(missing)} baseline entries not seen in this run; "
"run other bench files in the same crate to cover them)"
)
if regressions or (missing and strict_missing):
print(
f"FAIL: {len(regressions)} regression(s), "
f"{len(missing) if strict_missing else 0} missing bench(es)"
)
return 1
print(
f"OK: {len(passes)} bench(es) within budget "
f"(default +{budget:.1f}%, per-bench overrides where noted)"
)
return 0
if __name__ == "__main__":
sys.exit(main())
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
DEFAULT_CRATES=$(cat <<'ENTRIES'
fluxer_desktop/native/rt-thread:tick
fluxer_desktop/native/audio-mix:mix
fluxer_desktop/native/screen-frame-bus:staging
fluxer_desktop/native/screen-frame-bus:frame_pool
fluxer_desktop/native/nv12-gpu-pack:pack
fluxer_desktop/native/encoder-ring:ring
fluxer_desktop/native/linux-audio-capture:end_to_end
fluxer_desktop/native/linux-screen-capture:pipewire_callback
fluxer_desktop/native/webrtc-sender:frame_bus
fluxer_desktop/native/rust:native_core
ENTRIES
)
ENTRIES="${BENCH_CRATES:-$DEFAULT_CRATES}"
log() { printf '[check-all] %s\n' "$*" >&2; }
pass_count=0
fail_count=0
skip_count=0
failed_names=()
while IFS= read -r entry; do
[ -n "$entry" ] || continue
case "$entry" in
\#*) continue;;
esac
crate="${entry%%:*}"
bench="${entry##*:}"
if [ "${BENCH_SKIP_GPU:-0}" = "1" ] && [ "$crate" = "fluxer_desktop/native/nv12-gpu-pack" ]; then
log "skip (gpu): $crate $bench"
skip_count=$((skip_count + 1))
continue
fi
log "==> $crate :: $bench"
if "$SCRIPT_DIR/check-regression.sh" "$REPO_ROOT/$crate" "$bench"; then
pass_count=$((pass_count + 1))
else
fail_count=$((fail_count + 1))
failed_names+=("$crate::$bench")
fi
done <<< "$ENTRIES"
printf '\n[check-all] summary: %d passed, %d failed, %d skipped\n' \
"$pass_count" "$fail_count" "$skip_count"
if [ "$fail_count" -gt 0 ]; then
printf '[check-all] failed entries:\n'
for name in "${failed_names[@]}"; do
printf ' - %s\n' "$name"
done
exit 1
fi
exit 0
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -euo pipefail
BUDGET="${BENCH_REGRESSION_BUDGET_PERCENT:-5}"
MEASUREMENT_TIME="${BENCH_MEASUREMENT_TIME:-5}"
WARM_UP_TIME="${BENCH_WARM_UP_TIME:-2}"
log() { printf '[check-regression] %s\n' "$*" >&2; }
fail() { log "ERROR: $*"; exit 1; }
if [ "$#" -ne 2 ]; then
fail "usage: $0 <crate-path> <bench-name>"
fi
CRATE_PATH="$1"
BENCH_NAME="$2"
[ -d "$CRATE_PATH" ] || fail "crate path not a directory: $CRATE_PATH"
[ -f "$CRATE_PATH/Cargo.toml" ] || fail "no Cargo.toml in $CRATE_PATH"
BASELINE="$CRATE_PATH/benches/baseline.json"
[ -f "$BASELINE" ] || fail "no baseline.json at $BASELINE"
command -v cargo >/dev/null || fail "cargo not on PATH"
command -v python3 >/dev/null || fail "python3 not on PATH"
LOG_FILE="$(mktemp -t bench-regression.XXXXXX.log)"
trap 'rm -f "$LOG_FILE"' EXIT
log "running cargo bench --bench $BENCH_NAME in $CRATE_PATH"
(
cd "$CRATE_PATH"
cargo bench --bench "$BENCH_NAME" -- \
--warm-up-time "$WARM_UP_TIME" \
--measurement-time "$MEASUREMENT_TIME"
) >"$LOG_FILE" 2>&1 || {
tail -50 "$LOG_FILE" >&2
fail "cargo bench failed; see log above"
}
BUDGET_PERCENT="$BUDGET" BASELINE_PATH="$BASELINE" \
python3 "$(dirname "$0")/_compare_criterion.py" "$LOG_FILE"
+3436
View File
File diff suppressed because it is too large Load Diff
+26
View File
@@ -0,0 +1,26 @@
[package]
name = "fluxer-ci"
version = "0.1.0"
edition = "2024"
license = "AGPL-3.0-or-later"
[dependencies]
anyhow = "1.0.102"
aws-config = "1.8.18"
aws-sdk-s3 = "1.135.0"
base64 = "0.22.1"
bytes = "1.11.1"
chrono = "0.4.45"
clap = { version = "4.6.1", features = ["derive"] }
flate2 = "1.1.9"
hex = "0.4.3"
md-5 = "0.11.0"
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.150"
sha2 = "0.11.0"
tar = "0.4.46"
tempfile = "3.27.0"
tokio = { version = "1.52.3", features = ["fs", "io-util", "macros", "process", "rt-multi-thread", "signal", "sync", "time"] }
walkdir = "2.5.0"
zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
+539
View File
@@ -0,0 +1,539 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::app_wasm::resolve_app_dir;
use anyhow::{Context, Result, bail, ensure};
use clap::Args;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsStr;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::process::{Child, Command};
use tokio::sync::watch;
use tokio::time::timeout;
use walkdir::WalkDir;
const DEFAULT_SKIP_DIRS: &[&str] = &[".git", "node_modules", "dist", "target", "pkg", "pkgs"];
#[derive(Debug, Args, Clone)]
pub struct AppDevServerArgs {
#[arg(long)]
app_dir: Option<PathBuf>,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct StepMetadata {
last_run: f64,
inputs: BTreeMap<String, f64>,
}
type Metadata = BTreeMap<String, StepMetadata>;
pub async fn run(args: AppDevServerArgs) -> Result<()> {
let project_root = args.app_dir.unwrap_or(resolve_app_dir()?);
let mut server = AppDevServer::new(project_root);
server.run().await
}
struct AppDevServer {
project_root: PathBuf,
metadata_file: PathBuf,
metadata: Metadata,
}
impl AppDevServer {
fn new(project_root: PathBuf) -> Self {
Self {
metadata_file: project_root.join(".devserver-cache.json"),
project_root,
metadata: Metadata::default(),
}
}
async fn run(&mut self) -> Result<()> {
self.load_metadata();
let (shutdown_tx, mut shutdown_rx) = watch::channel(false);
tokio::spawn(listen_for_shutdown(shutdown_tx));
self.run_cached_step(
"wasm",
gather_wasm_inputs,
"pnpm wasm:codegen",
|server, shutdown| Box::pin(server.run_command("pnpm", &["wasm:codegen"], shutdown)),
&mut shutdown_rx,
)
.await?;
self.run_cached_step(
"colors",
gather_color_inputs,
"pnpm generate:colors",
|server, shutdown| Box::pin(server.run_command("pnpm", &["generate:colors"], shutdown)),
&mut shutdown_rx,
)
.await?;
self.run_cached_step(
"messageLayout",
gather_message_layout_inputs,
"pnpm generate:message-layout",
|server, shutdown| {
Box::pin(server.run_command("pnpm", &["generate:message-layout"], shutdown))
},
&mut shutdown_rx,
)
.await?;
self.run_cached_step(
"masks",
gather_mask_inputs,
"pnpm generate:masks",
|server, shutdown| Box::pin(server.run_command("pnpm", &["generate:masks"], shutdown)),
&mut shutdown_rx,
)
.await?;
self.run_cached_step(
"cssTypes",
gather_css_module_inputs,
"pnpm generate:css-types",
|server, shutdown| {
Box::pin(server.run_command("pnpm", &["generate:css-types"], shutdown))
},
&mut shutdown_rx,
)
.await?;
if env_truthy("FLUXER_APP_SKIP_I18N_COMPILE") {
eprintln!("Skipping pnpm lingui:compile because FLUXER_APP_SKIP_I18N_COMPILE is set.");
} else {
self.run_cached_step(
"lingui",
gather_lingui_inputs,
"pnpm lingui:compile",
|server, shutdown| {
Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown))
},
&mut shutdown_rx,
)
.await?;
}
if *shutdown_rx.borrow() {
return Ok(());
}
self.clean_dist()?;
let mut css_type_watcher = self.start_css_type_watcher()?;
let rspack_result = self.run_rspack(&mut shutdown_rx).await;
terminate_child(&mut css_type_watcher).await;
rspack_result
}
fn load_metadata(&mut self) {
match fs::read_to_string(&self.metadata_file) {
Ok(raw) => match serde_json::from_str::<Metadata>(&raw) {
Ok(metadata) => {
self.metadata = metadata;
}
Err(error) => {
eprintln!(
"Failed to parse dev server metadata cache, falling back to full rebuild: {error}"
);
self.metadata = Metadata::default();
}
},
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
self.metadata = Metadata::default();
}
Err(error) => {
eprintln!(
"Failed to read dev server metadata cache, falling back to full rebuild: {error}"
);
self.metadata = Metadata::default();
}
}
}
fn save_metadata(&self) -> Result<()> {
if let Some(parent) = self.metadata_file.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
fs::write(
&self.metadata_file,
serde_json::to_string_pretty(&self.metadata)?,
)
.with_context(|| format!("Failed to write {}", self.metadata_file.display()))
}
async fn run_cached_step<G, E>(
&mut self,
step_name: &'static str,
gather_inputs: G,
label: &'static str,
execute: E,
shutdown: &mut watch::Receiver<bool>,
) -> Result<()>
where
G: Fn(&Path) -> Result<BTreeMap<String, f64>>,
E: for<'a> FnOnce(
&'a AppDevServer,
&'a mut watch::Receiver<bool>,
)
-> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>>,
{
let inputs = gather_inputs(&self.project_root)?;
if !self.should_run_step(step_name, &inputs) {
println!("Skipping {label} (no changes detected)");
return Ok(());
}
execute(self, shutdown).await?;
self.metadata.insert(
step_name.to_string(),
StepMetadata {
last_run: timestamp_ms(SystemTime::now())?,
inputs,
},
);
self.save_metadata()
}
fn should_run_step(&self, step_name: &str, inputs: &BTreeMap<String, f64>) -> bool {
let Some(entry) = self.metadata.get(step_name) else {
return true;
};
&entry.inputs != inputs
}
async fn run_command(
&self,
command: &str,
args: &[&str],
shutdown: &mut watch::Receiver<bool>,
) -> Result<()> {
if *shutdown.borrow() {
return Ok(());
}
let mut child = spawn_child(command, args, &self.project_root)?;
let status = wait_for_child(command, args, &mut child, shutdown).await?;
if *shutdown.borrow() {
return Ok(());
}
ensure!(
status.success(),
"{} exited with status {}",
display_command(command, args),
status.code().unwrap_or(1)
);
Ok(())
}
fn clean_dist(&self) -> Result<()> {
let dist_path = self.project_root.join("dist");
match fs::remove_dir_all(&dist_path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => {
Err(error).with_context(|| format!("Failed to remove {}", dist_path.display()))
}
}
}
fn start_css_type_watcher(&self) -> Result<Child> {
let tcm = self.project_root.join("node_modules/.bin/tcm");
println!(
"+ {} src --pattern '**/*.module.css' --watch --silent",
tcm.display()
);
Command::new(tcm)
.args(["src", "--pattern", "**/*.module.css", "--watch", "--silent"])
.current_dir(&self.project_root)
.stdin(Stdio::null())
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.spawn()
.context("Failed to start CSS type watcher")
}
async fn run_rspack(&self, shutdown: &mut watch::Receiver<bool>) -> Result<()> {
let rspack = self.project_root.join("node_modules/.bin/rspack");
let rspack_string = rspack.to_string_lossy().to_string();
let mut child = spawn_child(
&rspack_string,
&["serve", "--mode", "development"],
&self.project_root,
)?;
let status = wait_for_child(
&rspack_string,
&["serve", "--mode", "development"],
&mut child,
shutdown,
)
.await?;
if *shutdown.borrow() {
return Ok(());
}
if status.success() {
Ok(())
} else {
bail!(
"rspack serve exited with status {}",
status.code().unwrap_or(1)
)
}
}
}
fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap<String, f64>> {
let mut result = BTreeMap::new();
for rel_path in paths {
let absolute_path = project_root.join(rel_path);
let metadata = fs::metadata(&absolute_path)
.with_context(|| format!("Failed to stat {}", absolute_path.display()))?;
ensure!(
metadata.is_file(),
"Expected {} to be a file when collecting dev server cache inputs.",
rel_path.display()
);
result.insert(rel_path_key(rel_path), timestamp_ms(metadata.modified()?)?);
}
Ok(result)
}
fn collect_directory_stats<P>(
project_root: &Path,
root_rel: &Path,
predicate: P,
) -> Result<BTreeMap<String, f64>>
where
P: Fn(&str) -> bool,
{
let skip_dirs: BTreeSet<&str> = DEFAULT_SKIP_DIRS.iter().copied().collect();
let root = project_root.join(root_rel);
let mut result = BTreeMap::new();
if !root.exists() {
return Ok(result);
}
for entry in WalkDir::new(&root)
.into_iter()
.filter_entry(|entry| should_walk_entry(entry.path(), &skip_dirs))
{
let entry = entry.with_context(|| format!("Failed to read {}", root.display()))?;
if !entry.file_type().is_file() {
continue;
}
let rel_from_root = entry
.path()
.strip_prefix(&root)
.with_context(|| format!("Failed to relativize {}", entry.path().display()))?
.to_path_buf();
let rel_path = root_rel.join(rel_from_root);
let key = rel_path_key(&rel_path);
if !predicate(&key) {
continue;
}
result.insert(key, timestamp_ms(entry.metadata()?.modified()?)?);
}
Ok(result)
}
fn should_walk_entry(path: &Path, skip_dirs: &BTreeSet<&str>) -> bool {
path.file_name()
.and_then(OsStr::to_str)
.is_none_or(|name| !skip_dirs.contains(name))
}
fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
let markdown_parser_rust_dir = PathBuf::from("../packages/markdown_parser/rust");
let mut inputs = collect_file_stats(
project_root,
&[
PathBuf::from("../tools/ci/Cargo.toml"),
PathBuf::from("../tools/ci/src/app_dev_server.rs"),
PathBuf::from("../tools/ci/src/app_wasm.rs"),
PathBuf::from("../tools/ci/src/common.rs"),
PathBuf::from("../tools/ci/src/lib.rs"),
PathBuf::from("../tools/ci/templates/libfluxcore_wrapper.js"),
PathBuf::from("../tools/ci/templates/libfluxcore_wrapper.d.ts"),
markdown_parser_rust_dir.join("Cargo.toml"),
],
)?;
inputs.extend(collect_directory_stats(
project_root,
Path::new("rust/libfluxcore"),
|path| !path.contains("/target/"),
)?);
inputs.extend(collect_directory_stats(
project_root,
&markdown_parser_rust_dir,
|path| !path.contains("/target/"),
)?);
Ok(inputs)
}
fn gather_color_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_file_stats(
project_root,
&[PathBuf::from("scripts/GenerateColorSystem.ts")],
)
}
fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_file_stats(
project_root,
&[
PathBuf::from("scripts/GenerateMessageLayoutCss.ts"),
PathBuf::from("src/features/theme/layout/MessageLayoutSpec.ts"),
],
)
}
fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_file_stats(
project_root,
&[
PathBuf::from("scripts/GenerateAvatarMasks.ts"),
PathBuf::from("src/features/ui/constants/TypingConstants.ts"),
],
)
}
fn gather_css_module_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_directory_stats(project_root, Path::new("src"), |path| {
path.ends_with(".module.css")
})
}
fn gather_lingui_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
collect_directory_stats(
project_root,
Path::new("src/features/i18n/locales"),
|path| path.ends_with(".po"),
)
}
fn spawn_child(command: &str, args: &[&str], cwd: &Path) -> Result<Child> {
println!("+ {}", display_command(command, args));
Command::new(command)
.args(args)
.current_dir(cwd)
.stdin(Stdio::inherit())
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.spawn()
.with_context(|| format!("Failed to run {}", display_command(command, args)))
}
async fn wait_for_child(
command: &str,
args: &[&str],
child: &mut Child,
shutdown: &mut watch::Receiver<bool>,
) -> Result<std::process::ExitStatus> {
tokio::select! {
status = child.wait() => {
status.with_context(|| format!("Failed to wait for {}", display_command(command, args)))
}
changed = shutdown.changed() => {
let _ = changed;
terminate_child(child).await;
child.wait().await.with_context(|| format!("Failed to wait for {}", display_command(command, args)))
}
}
}
async fn terminate_child(child: &mut Child) {
if child.id().is_none() {
return;
}
let _ = child.start_kill();
let _ = timeout(Duration::from_secs(5), child.wait()).await;
}
async fn listen_for_shutdown(shutdown_tx: watch::Sender<bool>) {
let signal = wait_for_shutdown_signal().await;
println!("\nReceived {signal}, shutting down fluxer app dev server...");
let _ = shutdown_tx.send(true);
}
#[cfg(unix)]
async fn wait_for_shutdown_signal() -> &'static str {
let mut sigterm = match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
{
Ok(signal) => signal,
Err(_) => {
let _ = tokio::signal::ctrl_c().await;
return "SIGINT";
}
};
tokio::select! {
_ = tokio::signal::ctrl_c() => "SIGINT",
_ = sigterm.recv() => "SIGTERM",
}
}
#[cfg(not(unix))]
async fn wait_for_shutdown_signal() -> &'static str {
let _ = tokio::signal::ctrl_c().await;
"SIGINT"
}
fn rel_path_key(path: &Path) -> String {
path.to_string_lossy().replace('\\', "/")
}
fn timestamp_ms(timestamp: SystemTime) -> Result<f64> {
Ok(timestamp
.duration_since(UNIX_EPOCH)
.context("File timestamp predates UNIX epoch")?
.as_secs_f64()
* 1000.0)
}
fn env_truthy(name: &str) -> bool {
std::env::var(name)
.ok()
.is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true"))
}
fn display_command(command: &str, args: &[&str]) -> String {
std::iter::once(command.to_string())
.chain(args.iter().map(|arg| quote_arg(arg)))
.collect::<Vec<_>>()
.join(" ")
}
fn quote_arg(arg: &str) -> String {
if arg
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.' | '/' | ':' | '='))
{
arg.to_string()
} else {
format!("{arg:?}")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rel_path_keys_are_posix_like() {
assert_eq!(
rel_path_key(Path::new("scripts/GenerateColorSystem.ts")),
"scripts/GenerateColorSystem.ts"
);
}
#[test]
fn display_command_quotes_globs() {
assert_eq!(
display_command("tcm", &["src", "--pattern", "**/*.module.css"]),
"tcm src --pattern \"**/*.module.css\""
);
}
}
+528
View File
@@ -0,0 +1,528 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::{
CALVER_SCHEME, CalverEnv, CommandSpec, S3UploadPlanItem, append_github_env,
append_github_output, collect_files, path_to_s3_key, require_env, resolve_calver, run_command,
runner_temp, s3_client, trim_option, upload_s3_plan_append_only,
};
use anyhow::{Context, Result, anyhow, ensure};
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64;
use chrono::Utc;
use clap::{Args, ValueEnum};
use serde_json::{Map, Value, json};
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
const DEFAULT_PUBLIC_ASSET_BASE_URL: &str = "https://fluxerstatic.com";
const DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED: &str = "true";
const DEFAULT_STATIC_BUCKET: &str = "fluxer-static";
const DEFAULT_S3_ENDPOINT: &str = "https://ewr1.vultrobjects.com";
const IMMUTABLE_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable";
#[derive(Debug, Args, Clone)]
pub struct BuildAppProxyArgs {
#[arg(long, value_enum)]
step: AppProxyStep,
#[arg(long)]
build_version: Option<String>,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "snake_case")]
enum AppProxyStep {
SetMetadata,
PrepareDockerConfig,
ConfigureGhcrAuth,
BuildAndExtract,
GenerateAssetManifest,
UploadAssets,
}
pub async fn run(args: BuildAppProxyArgs) -> Result<()> {
match args.step {
AppProxyStep::SetMetadata => set_metadata_step(args.build_version.as_deref()),
AppProxyStep::PrepareDockerConfig => prepare_docker_config_step(),
AppProxyStep::ConfigureGhcrAuth => configure_ghcr_auth_step(),
AppProxyStep::BuildAndExtract => build_and_extract_step(),
AppProxyStep::GenerateAssetManifest => generate_asset_manifest_step(),
AppProxyStep::UploadAssets => upload_assets_step().await,
}
}
fn set_metadata_step(build_version_arg: Option<&str>) -> Result<()> {
let calver_env = CalverEnv {
build_version: trim_option(build_version_arg.map(ToOwned::to_owned))
.or_else(|| trim_option(env::var("BUILD_VERSION").ok())),
fluxer_build_version: trim_option(env::var("FLUXER_BUILD_VERSION").ok()),
fluxer_build_date: trim_option(env::var("FLUXER_BUILD_DATE").ok()),
};
let version = resolve_calver(&calver_env, Utc::now())?;
append_github_output(&[
("build_version", version.as_str()),
("version", version.as_str()),
("calver_scheme", CALVER_SCHEME),
])
}
fn prepare_docker_config_step() -> Result<()> {
let docker_config = runner_temp().join("docker-config");
fs::create_dir_all(&docker_config)
.with_context(|| format!("Failed to create {}", docker_config.display()))?;
append_github_env(&[("DOCKER_CONFIG", docker_config.to_string_lossy().as_ref())])
}
fn configure_ghcr_auth_step() -> Result<()> {
let docker_config = require_env("DOCKER_CONFIG")?;
let username = require_env("GHCR_USERNAME")?;
let token = require_env("GHCR_TOKEN")?;
let path = PathBuf::from(docker_config).join("config.json");
write_ghcr_auth_config(&path, &username, &token)
}
fn write_ghcr_auth_config(path: &Path, username: &str, token: &str) -> Result<()> {
let mut config = if path.exists() {
serde_json::from_str::<Value>(
&fs::read_to_string(path)
.with_context(|| format!("Failed to read {}", path.display()))?,
)
.with_context(|| format!("Failed to parse {}", path.display()))?
} else {
Value::Object(Map::new())
};
let root = config
.as_object_mut()
.ok_or_else(|| anyhow!("Docker config root must be a JSON object"))?;
let auths = root
.entry("auths")
.or_insert_with(|| Value::Object(Map::new()))
.as_object_mut()
.ok_or_else(|| anyhow!("Docker config auths must be a JSON object"))?;
auths.insert(
"ghcr.io".to_string(),
json!({ "auth": ghcr_auth_value(username, token) }),
);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
fs::write(path, format!("{}\n", serde_json::to_string(&config)?))
.with_context(|| format!("Failed to write {}", path.display()))
}
fn ghcr_auth_value(username: &str, token: &str) -> String {
BASE64.encode(format!("{username}:{token}"))
}
fn build_and_extract_step() -> Result<()> {
run_command(build_and_extract_command()?)
}
fn build_and_extract_command() -> Result<CommandSpec> {
let build_version = require_env("BUILD_VERSION")?;
let public_asset_base_url = env::var("PUBLIC_ASSET_BASE_URL")
.unwrap_or_else(|_| DEFAULT_PUBLIC_ASSET_BASE_URL.to_string());
let image_repo = match env::var("IMAGE_REPO") {
Ok(value) => value,
Err(_) => format!("ghcr.io/{}/fluxer-app-proxy", ghcr_owner()?),
};
Ok(CommandSpec::new("docker")
.args(["buildx", "bake", "-f", "fluxer_app_proxy/docker-bake.hcl"])
.env("IMAGE_REPO", image_repo)
.env("BUILD_VERSION", build_version)
.env("PUBLIC_ASSET_BASE_URL", public_asset_base_url)
.env(
"FLUXER_APP_PROXY_TIME_FREEZE_ENABLED",
env::var("FLUXER_APP_PROXY_TIME_FREEZE_ENABLED")
.unwrap_or_else(|_| DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED.to_string()),
)
.env(
"CACHE_FROM",
env::var("CACHE_FROM")
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy".to_string()),
)
.env(
"CACHE_TO",
env::var("CACHE_TO")
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy,mode=max".to_string()),
)
.env(
"DOCKER_BUILD_SUMMARY",
env::var("DOCKER_BUILD_SUMMARY").unwrap_or_else(|_| "false".to_string()),
)
.env(
"DOCKER_BUILD_RECORD_UPLOAD",
env::var("DOCKER_BUILD_RECORD_UPLOAD").unwrap_or_else(|_| "false".to_string()),
))
}
fn ghcr_owner() -> Result<String> {
for key in ["GHCR_OWNER", "GITHUB_REPOSITORY_OWNER", "OWNER"] {
if let Ok(value) = env::var(key) {
let value = value.trim();
if !value.is_empty() {
return Ok(value.to_string());
}
}
}
if let Ok(repository) = env::var("GITHUB_REPOSITORY") {
if let Some((owner, _)) = repository.split_once('/') {
let owner = owner.trim();
if !owner.is_empty() {
return Ok(owner.to_string());
}
}
}
Err(anyhow!(
"GHCR owner must be set with GHCR_OWNER, GITHUB_REPOSITORY_OWNER, OWNER, or GITHUB_REPOSITORY"
))
}
fn generate_asset_manifest_step() -> Result<()> {
let dist = app_dist_dir();
let manifest_path = dist.join("assets-manifest.txt");
let assets = asset_manifest_entries(&dist)?;
fs::write(&manifest_path, format!("{}\n", assets.join("\n")))
.with_context(|| format!("Failed to write {}", manifest_path.display()))?;
println!("=== asset manifest ===");
for asset in &assets {
println!("{asset}");
}
println!("total assets: {}", assets.len());
Ok(())
}
fn app_dist_dir() -> PathBuf {
env::var("APP_DIST_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("app-dist-output/dist"))
}
fn asset_manifest_entries(dist: &Path) -> Result<Vec<String>> {
let assets_dir = dist.join("assets");
ensure!(
assets_dir.exists(),
"App proxy assets directory is missing: {}",
assets_dir.display()
);
let mut entries = collect_files(&assets_dir)?
.into_iter()
.filter(|path| !is_source_map_asset(path))
.map(|path| {
path.strip_prefix(dist)
.with_context(|| format!("Failed to relativize {}", path.display()))
.map(path_to_s3_key)
})
.collect::<Result<Vec<_>>>()?;
entries.sort();
Ok(entries)
}
fn is_source_map_asset(path: &Path) -> bool {
path.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("map"))
}
async fn upload_assets_step() -> Result<()> {
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
let bucket = env::var("STATIC_BUCKET").unwrap_or_else(|_| DEFAULT_STATIC_BUCKET.to_string());
let dist = app_dist_dir();
let manifest_path = dist.join("assets-manifest.txt");
let assets = read_asset_manifest(&manifest_path)?;
ensure!(!assets.is_empty(), "{} is empty", manifest_path.display());
let plan = asset_upload_plan(&dist, &assets)?;
let stats = upload_s3_plan_append_only(&client, &bucket, plan).await?;
println!("upload complete - {} assets", assets.len());
println!(
"append-only result - uploaded {}, skipped existing {}, repaired metadata {}",
stats.uploaded, stats.skipped_existing, stats.metadata_repaired
);
Ok(())
}
fn asset_upload_plan(dist: &Path, assets: &[String]) -> Result<Vec<S3UploadPlanItem>> {
assets
.iter()
.map(|asset| {
let path = dist.join(asset);
ensure!(
path.is_file(),
"Manifest asset is missing: {}",
path.display()
);
Ok(S3UploadPlanItem::new(path, asset.clone())
.with_detected_content_type()
.with_cache_control(IMMUTABLE_ASSET_CACHE_CONTROL)
.repair_existing_metadata())
})
.collect()
}
fn read_asset_manifest(path: &Path) -> Result<Vec<String>> {
let manifest =
fs::read_to_string(path).with_context(|| format!("Failed to read {}", path.display()))?;
manifest
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.map(validate_manifest_asset)
.collect()
}
fn validate_manifest_asset(asset: &str) -> Result<String> {
ensure!(
asset.starts_with("assets/"),
"Asset manifest entry must be under assets/: {asset}"
);
ensure!(
!asset.contains("..") && !asset.starts_with('/') && !asset.contains('\\'),
"Invalid asset manifest path: {asset}"
);
Ok(asset.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::common::parse_version_instant;
use chrono::{DateTime, TimeZone, Utc};
use std::ffi::OsString;
fn dt(year: i32, month: u32, day: u32, hour: u32, minute: u32, second: u32) -> DateTime<Utc> {
Utc.with_ymd_and_hms(year, month, day, hour, minute, second)
.single()
.unwrap()
}
fn write_file(path: &Path, contents: &str) {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(path, contents).unwrap();
}
#[test]
fn resolves_calver_from_explicit_or_date_override() {
let explicit = CalverEnv {
build_version: Some("2026.520.1".to_string()),
fluxer_build_version: Some("2026.521.2".to_string()),
fluxer_build_date: Some("2026-05-22T03:04:05Z".to_string()),
};
assert_eq!(
resolve_calver(&explicit, dt(2026, 1, 1, 0, 0, 0)).unwrap(),
"2026.520.1"
);
let generated = CalverEnv {
fluxer_build_date: Some("2026-05-20T01:02:03Z".to_string()),
..CalverEnv::default()
};
assert_eq!(
resolve_calver(&generated, dt(2026, 1, 1, 0, 0, 0)).unwrap(),
"2026.520.10203"
);
}
#[test]
fn rejects_invalid_calver_time() {
assert_eq!(
parse_version_instant("2026.520.246000")
.unwrap_err()
.to_string(),
"Invalid build version date/time: 2026.520.246000"
);
}
#[test]
fn ghcr_auth_config_merges_existing_auths() {
let temp = tempfile::tempdir().unwrap();
let config_path = temp.path().join("config.json");
fs::write(
&config_path,
r#"{"auths":{"example.com":{"auth":"old"}},"currentContext":"builder"}"#,
)
.unwrap();
write_ghcr_auth_config(&config_path, "octo", "secret").unwrap();
let config: Value =
serde_json::from_str(&fs::read_to_string(config_path).unwrap()).unwrap();
assert_eq!(config["auths"]["example.com"]["auth"], "old");
assert_eq!(
config["auths"]["ghcr.io"]["auth"],
BASE64.encode("octo:secret")
);
assert_eq!(config["currentContext"], "builder");
}
#[test]
fn ghcr_auth_config_rejects_non_object_roots_and_auths() {
let temp = tempfile::tempdir().unwrap();
let config_path = temp.path().join("config.json");
fs::write(&config_path, "[]").unwrap();
assert_eq!(
write_ghcr_auth_config(&config_path, "octo", "secret")
.unwrap_err()
.to_string(),
"Docker config root must be a JSON object"
);
fs::write(&config_path, r#"{"auths":[]}"#).unwrap();
assert_eq!(
write_ghcr_auth_config(&config_path, "octo", "secret")
.unwrap_err()
.to_string(),
"Docker config auths must be a JSON object"
);
}
#[test]
fn build_command_sets_bake_environment() {
let command = CommandSpec::new("docker")
.args(["buildx", "bake", "-f", "fluxer_app_proxy/docker-bake.hcl"])
.env("IMAGE_REPO", "ghcr.io/example/fluxer-app-proxy")
.env("BUILD_VERSION", "2026.520.1")
.env("PUBLIC_ASSET_BASE_URL", DEFAULT_PUBLIC_ASSET_BASE_URL)
.env(
"FLUXER_APP_PROXY_TIME_FREEZE_ENABLED",
DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED,
);
assert_eq!(command.program, OsString::from("docker"));
assert_eq!(
command.args,
vec![
OsString::from("buildx"),
OsString::from("bake"),
OsString::from("-f"),
OsString::from("fluxer_app_proxy/docker-bake.hcl"),
]
);
assert!(command.env.contains(&(
OsString::from("BUILD_VERSION"),
OsString::from("2026.520.1")
)));
assert!(command.env.contains(&(
OsString::from("FLUXER_APP_PROXY_TIME_FREEZE_ENABLED"),
OsString::from(DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED)
)));
}
#[test]
fn asset_manifest_entries_are_sorted_and_relative_to_dist() {
let temp = tempfile::tempdir().unwrap();
let dist = temp.path().join("dist");
write_file(&dist.join("assets/z.js"), "z");
write_file(&dist.join("assets/z.js.map"), "{}");
write_file(&dist.join("assets/chunks/a.js"), "a");
write_file(&dist.join("assets/chunks/a.js.map"), "{}");
write_file(&dist.join("index.html"), "ignored");
assert_eq!(
asset_manifest_entries(&dist).unwrap(),
vec!["assets/chunks/a.js", "assets/z.js"]
);
}
#[test]
fn asset_manifest_entries_require_assets_directory() {
let temp = tempfile::tempdir().unwrap();
let dist = temp.path().join("dist");
fs::create_dir_all(&dist).unwrap();
assert!(
asset_manifest_entries(&dist)
.unwrap_err()
.to_string()
.contains("App proxy assets directory is missing")
);
}
#[test]
fn manifest_reader_trims_blank_lines_and_keeps_order() {
let temp = tempfile::tempdir().unwrap();
let manifest = temp.path().join("assets-manifest.txt");
fs::write(&manifest, "\n assets/b.js \n\nassets/a.js\n").unwrap();
assert_eq!(
read_asset_manifest(&manifest).unwrap(),
vec!["assets/b.js", "assets/a.js"]
);
}
#[test]
fn asset_upload_plan_preserves_manifest_keys() {
let temp = tempfile::tempdir().unwrap();
let dist = temp.path().join("dist");
write_file(&dist.join("assets/a.js"), "a");
write_file(&dist.join("assets/chunks/b.js"), "b");
let assets = vec!["assets/a.js".to_string(), "assets/chunks/b.js".to_string()];
let plan = asset_upload_plan(&dist, &assets).unwrap();
assert_eq!(
plan.iter()
.map(|item| item.key.as_str())
.collect::<Vec<_>>(),
vec!["assets/a.js", "assets/chunks/b.js"]
);
assert_eq!(plan[0].path, dist.join("assets/a.js"));
assert_eq!(plan[1].path, dist.join("assets/chunks/b.js"));
assert_eq!(
plan[0].content_type.as_deref(),
Some("application/javascript; charset=utf-8")
);
assert_eq!(
plan[0].cache_control.as_deref(),
Some(IMMUTABLE_ASSET_CACHE_CONTROL)
);
assert!(plan[0].repair_existing_metadata);
}
#[test]
fn asset_upload_plan_rejects_manifest_entries_missing_on_disk() {
let temp = tempfile::tempdir().unwrap();
let dist = temp.path().join("dist");
fs::create_dir_all(&dist).unwrap();
let assets = vec!["assets/missing.js".to_string()];
assert!(
asset_upload_plan(&dist, &assets)
.unwrap_err()
.to_string()
.contains("Manifest asset is missing")
);
}
#[test]
fn manifest_reader_rejects_paths_outside_assets() {
let temp = tempfile::tempdir().unwrap();
let manifest = temp.path().join("assets-manifest.txt");
fs::write(&manifest, "assets/a.js\n../secret\n").unwrap();
assert!(read_asset_manifest(&manifest).is_err());
}
#[test]
fn manifest_reader_rejects_absolute_parent_and_backslash_paths() {
for asset in ["/assets/a.js", "assets/../secret", r"assets\app.js"] {
assert!(validate_manifest_asset(asset).is_err(), "{asset}");
}
}
#[test]
fn path_to_s3_key_uses_forward_slashes() {
assert_eq!(
path_to_s3_key(Path::new("assets").join("chunks").join("a.js").as_path()),
"assets/chunks/a.js"
);
}
}
+386
View File
@@ -0,0 +1,386 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::{CommandSpec, command_succeeds, env_bool, output_text, run_command};
use anyhow::{Context, Result, anyhow, ensure};
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64;
use clap::Args;
use std::env;
use std::ffi::OsString;
use std::fs;
use std::path::{Path, PathBuf};
use tempfile::TempDir;
const LIBFLUXCORE_WASM_BINDGEN_VERSION: &str = "0.2.122";
const LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES: u64 = 300 * 1024;
const LIBFLUXCORE_WRAPPER_JS: &str = include_str!("../templates/libfluxcore_wrapper.js");
const LIBFLUXCORE_WRAPPER_DTS: &str = include_str!("../templates/libfluxcore_wrapper.d.ts");
#[derive(Debug, Args, Clone)]
pub struct BuildAppWasmArgs {
#[arg(long)]
app_dir: Option<PathBuf>,
}
#[derive(Debug, Args, Clone)]
pub struct BuildMarkdownParserWasmArgs {
#[arg(long)]
app_dir: Option<PathBuf>,
}
pub fn run_build_app_wasm(args: BuildAppWasmArgs) -> Result<()> {
let app_dir = args.app_dir.unwrap_or(resolve_app_dir()?);
build_markdown_parser_wasm(&app_dir)?;
build_libfluxcore_wasm(&app_dir)
}
pub fn run_build_markdown_parser_wasm(args: BuildMarkdownParserWasmArgs) -> Result<()> {
let app_dir = args.app_dir.unwrap_or(resolve_app_dir()?);
build_markdown_parser_wasm(&app_dir)
}
fn build_markdown_parser_wasm(app_dir: &Path) -> Result<()> {
let rust_source_dir = app_dir.join("../packages/markdown_parser/rust");
let bytes_path =
app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts");
let temp = TempDir::new().context("Failed to create source temp directory")?;
let target_dir = temp.path().join("target");
run_command(
CommandSpec::new("cargo")
.args(["build", "--release", "--target", "wasm32-unknown-unknown"])
.env("CARGO_TARGET_DIR", target_dir.to_string_lossy().as_ref())
.current_dir(&rust_source_dir),
)?;
let wasm_path = target_dir.join("wasm32-unknown-unknown/release/fluxer_markdown_parser.wasm");
let wasm =
fs::read(&wasm_path).with_context(|| format!("Failed to read {}", wasm_path.display()))?;
let content = markdown_wasm_bytes_content(&wasm);
if let Some(parent) = bytes_path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
fs::write(&bytes_path, content)
.with_context(|| format!("Failed to write {}", bytes_path.display()))
}
fn build_libfluxcore_wasm(app_dir: &Path) -> Result<()> {
let rust_package_dir = app_dir.join("rust/libfluxcore");
let out_dir = app_dir.join("pkgs/libfluxcore");
let wasm_path = out_dir.join("libfluxcore_bg.wasm");
let previous_wasm_size = file_size(&wasm_path)?;
fs::create_dir_all(&out_dir)
.with_context(|| format!("Failed to create {}", out_dir.display()))?;
let mut build = CommandSpec::new("cargo")
.args([
"build",
"--release",
"--target",
"wasm32-unknown-unknown",
"--manifest-path",
])
.arg(rust_package_dir.join("Cargo.toml"))
.current_dir(&rust_package_dir);
if env_bool("FLUXCORE_WASM_SIMD") {
let rustflags = match env::var("RUSTFLAGS") {
Ok(value) if !value.trim().is_empty() => format!("{value} -C target-feature=+simd128"),
_ => "-C target-feature=+simd128".to_string(),
};
build = build.env("RUSTFLAGS", rustflags);
}
run_command(build)?;
let wasm_bindgen = ensure_wasm_bindgen_cli()?;
let temp = TempDir::new().context("Failed to create libfluxcore wasm-bindgen temp dir")?;
let bindgen_dir = temp.path().join("bindgen");
fs::create_dir_all(&bindgen_dir)
.with_context(|| format!("Failed to create {}", bindgen_dir.display()))?;
run_command(
CommandSpec::new(wasm_bindgen)
.args(["--target", "web", "--out-dir"])
.arg(&bindgen_dir)
.args(["--out-name", "libfluxcore"])
.arg(rust_package_dir.join("target/wasm32-unknown-unknown/release/libfluxcore.wasm"))
.current_dir(&rust_package_dir),
)?;
let bindgen_js_path = bindgen_dir.join("libfluxcore.js");
let bindgen_dts_path = bindgen_dir.join("libfluxcore.d.ts");
let bindgen_wasm_path = bindgen_dir.join("libfluxcore_bg.wasm");
let bindgen_wasm_dts_path = bindgen_dir.join("libfluxcore_bg.wasm.d.ts");
write_with_spdx(
&out_dir.join("libfluxcore_bindgen.js"),
&patch_libfluxcore_bindgen_js(
&fs::read_to_string(&bindgen_js_path)
.with_context(|| format!("Failed to read {}", bindgen_js_path.display()))?,
)?,
)?;
write_with_spdx(
&out_dir.join("libfluxcore_bindgen.d.ts"),
&patch_libfluxcore_bindgen_dts(
&fs::read_to_string(&bindgen_dts_path)
.with_context(|| format!("Failed to read {}", bindgen_dts_path.display()))?,
)?,
)?;
fs::copy(&bindgen_wasm_path, &wasm_path).with_context(|| {
format!(
"Failed to copy {} to {}",
bindgen_wasm_path.display(),
wasm_path.display()
)
})?;
fs::copy(
&bindgen_wasm_dts_path,
out_dir.join("libfluxcore_bg.wasm.d.ts"),
)
.with_context(|| format!("Failed to copy {}", bindgen_wasm_dts_path.display()))?;
fs::write(
out_dir.join("libfluxcore.js"),
libfluxcore_index_js_content(),
)
.with_context(|| {
format!(
"Failed to write {}",
out_dir.join("libfluxcore.js").display()
)
})?;
fs::write(
out_dir.join("libfluxcore.d.ts"),
libfluxcore_index_dts_content(),
)
.with_context(|| {
format!(
"Failed to write {}",
out_dir.join("libfluxcore.d.ts").display()
)
})?;
fs::write(
out_dir.join("package.json"),
libfluxcore_package_json_content(),
)
.with_context(|| format!("Failed to write {}", out_dir.join("package.json").display()))?;
fs::write(out_dir.join("README.md"), libfluxcore_readme_content())
.with_context(|| format!("Failed to write {}", out_dir.join("README.md").display()))?;
let wasm_size = file_size(&wasm_path)?
.ok_or_else(|| anyhow!("libfluxcore build did not emit {}", wasm_path.display()))?;
ensure!(
wasm_size <= LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES,
"libfluxcore_bg.wasm is {}, over the {} budget",
format_bytes(wasm_size),
format_bytes(LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES)
);
let size_comparison = match previous_wasm_size {
Some(previous) => format!("{} -> {}", format_bytes(previous), format_bytes(wasm_size)),
None => "no previous artifact".to_string(),
};
println!(
"libfluxcore_bg.wasm size: {size_comparison} (budget {})",
format_bytes(LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES)
);
Ok(())
}
fn patch_libfluxcore_bindgen_js(content: &str) -> Result<String> {
const MARKER: &str = "\nasync function __wbg_load(module, imports) {";
const RESET_EXPORT: &str = r#"
export function __resetLibfluxcoreWasmForMemoryPressure() {
wasmModule = undefined;
wasmInstance = undefined;
wasm = undefined;
cachedDataViewMemory0 = null;
cachedUint8ArrayMemory0 = null;
heap = new Array(1024).fill(undefined);
heap.push(undefined, null, true, false);
heap_next = heap.length;
numBytesDecoded = 0;
}
"#;
ensure!(
content.contains(MARKER),
"libfluxcore wasm-bindgen JS output did not contain reset insertion marker"
);
Ok(content.replacen(MARKER, &format!("{RESET_EXPORT}{MARKER}"), 1))
}
fn patch_libfluxcore_bindgen_dts(content: &str) -> Result<String> {
const MARKER: &str = "\nexport type InitInput";
const RESET_EXPORT: &str =
"\nexport function __resetLibfluxcoreWasmForMemoryPressure(): void;\n";
ensure!(
content.contains(MARKER),
"libfluxcore wasm-bindgen DTS output did not contain reset insertion marker"
);
Ok(content.replacen(MARKER, &format!("{RESET_EXPORT}{MARKER}"), 1))
}
pub(crate) fn resolve_app_dir() -> Result<PathBuf> {
let cwd = env::current_dir().context("Failed to resolve current directory")?;
if cwd.file_name().and_then(|value| value.to_str()) == Some("fluxer_app") {
return Ok(cwd);
}
if cwd.join("fluxer_app").is_dir() {
return Ok(cwd.join("fluxer_app"));
}
Err(anyhow!(
"Could not resolve fluxer_app directory from {}",
cwd.display()
))
}
fn ensure_wasm_bindgen_cli() -> Result<OsString> {
let expected = format!("wasm-bindgen {LIBFLUXCORE_WASM_BINDGEN_VERSION}");
if command_succeeds(CommandSpec::new("wasm-bindgen").arg("--version")) {
let version = output_text(CommandSpec::new("wasm-bindgen").arg("--version"))?;
if version.trim() == expected {
return Ok("wasm-bindgen".into());
}
}
run_command(CommandSpec::new("cargo").args([
"install",
"wasm-bindgen-cli",
"--version",
LIBFLUXCORE_WASM_BINDGEN_VERSION,
"--locked",
"--force",
]))?;
Ok("wasm-bindgen".into())
}
fn file_size(path: &Path) -> Result<Option<u64>> {
match fs::metadata(path) {
Ok(metadata) => Ok(Some(metadata.len())),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error).with_context(|| format!("Failed to stat {}", path.display())),
}
}
fn format_bytes(bytes: u64) -> String {
format!("{bytes} B")
}
fn write_with_spdx(path: &Path, content: &str) -> Result<()> {
fs::write(
path,
format!("// SPDX-License-Identifier: AGPL-3.0-or-later\n{content}"),
)
.with_context(|| format!("Failed to write {}", path.display()))
}
fn libfluxcore_index_js_content() -> String {
format!(
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
import {{crop_rotate_rgba_raw}} from './libfluxcore_bindgen.js';\n\
{LIBFLUXCORE_WRAPPER_JS}\n\
export * from './libfluxcore_bindgen.js';\n\
export {{default}} from './libfluxcore_bindgen.js';\n"
)
}
fn libfluxcore_index_dts_content() -> String {
format!(
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
export * from './libfluxcore_bindgen.js';\n\
export {{default}} from './libfluxcore_bindgen.js';\n\n\
{LIBFLUXCORE_WRAPPER_DTS}"
)
}
fn libfluxcore_package_json_content() -> String {
let manifest = serde_json::json!({
"name": "libfluxcore",
"private": true,
"type": "module",
"version": "0.0.0",
"license": "AGPL-3.0-or-later",
"sideEffects": false,
"files": [
"libfluxcore.js",
"libfluxcore.d.ts",
"libfluxcore_bindgen.js",
"libfluxcore_bindgen.d.ts",
"libfluxcore_bg.wasm",
"libfluxcore_bg.wasm.d.ts",
"README.md"
],
"main": "libfluxcore.js",
"module": "libfluxcore.js",
"types": "libfluxcore.d.ts",
"exports": {
".": {
"types": "./libfluxcore.d.ts",
"default": "./libfluxcore.js"
},
"./libfluxcore_bg.wasm": "./libfluxcore_bg.wasm"
}
});
format!("{manifest:#}\n")
}
fn libfluxcore_readme_content() -> &'static str {
"<!-- SPDX-License-Identifier: AGPL-3.0-or-later -->\n\
# libfluxcore\n\n\
Rust WebAssembly helpers and JavaScript codec wrappers for Fluxer media processing.\n"
}
fn markdown_wasm_bytes_content(wasm: &[u8]) -> String {
format!(
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
export const MARKDOWN_PARSER_WASM_BASE64 =\n\
\t'{}';\n",
BASE64.encode(wasm)
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn markdown_wasm_bytes_content_matches_legacy_node_output() {
assert_eq!(
markdown_wasm_bytes_content(b"hello"),
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
export const MARKDOWN_PARSER_WASM_BASE64 =\n\
\t'aGVsbG8=';\n"
);
}
#[test]
fn libfluxcore_index_reexports_bindgen_module() {
let content = libfluxcore_index_js_content();
assert!(content.contains("import {crop_rotate_rgba_raw} from './libfluxcore_bindgen.js';"));
assert!(content.contains("export * from './libfluxcore_bindgen.js';"));
assert!(content.contains("export function crop_rotate_rgba("));
}
#[test]
fn libfluxcore_bindgen_js_reset_hook_is_inserted() {
let content =
"function __wbg_finalize_init() {}\nasync function __wbg_load(module, imports) {}";
let patched = patch_libfluxcore_bindgen_js(content).expect("patch should succeed");
assert!(patched.contains("export function __resetLibfluxcoreWasmForMemoryPressure()"));
assert!(patched.contains("wasm = undefined;"));
assert!(patched.contains("async function __wbg_load(module, imports) {}"));
}
#[test]
fn libfluxcore_bindgen_dts_reset_hook_is_inserted() {
let content = "export function is_animated_image(input: Uint8Array): boolean;\nexport type InitInput = RequestInfo;";
let patched = patch_libfluxcore_bindgen_dts(content).expect("patch should succeed");
assert!(
patched.contains("export function __resetLibfluxcoreWasmForMemoryPressure(): void;")
);
assert!(patched.contains("export type InitInput = RequestInfo;"));
}
}
+143
View File
@@ -0,0 +1,143 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::{
CALVER_SCHEME, CalverEnv, append_github_env, append_github_output, parse_version_instant,
resolve_calver, trim_option,
};
use anyhow::Result;
use chrono::{Datelike, Timelike, Utc};
use clap::Args;
use std::env;
#[derive(Debug, Args, Clone)]
pub struct ResolveCalverArgs {
#[arg(long)]
github_output: bool,
#[arg(long)]
github_env: bool,
#[arg(long, default_value = "BUILD_VERSION")]
env_name: String,
}
pub fn run(args: ResolveCalverArgs) -> Result<()> {
let resolved = resolve_calver_from_env()?;
if args.github_output {
let output = calver_outputs(&resolved)?;
append_github_output(&[
("version", output.version.as_str()),
("build_version", output.version.as_str()),
("time", output.time.as_str()),
("micro", output.micro.as_str()),
("patch", output.micro.as_str()),
("date", output.date.as_str()),
("year", output.year.as_str()),
("month", output.month.as_str()),
("day", output.day.as_str()),
("month_day", output.month_day.as_str()),
("calver_scheme", CALVER_SCHEME),
])?;
}
if args.github_env {
append_github_env(&[(args.env_name.as_str(), resolved.as_str())])?;
}
println!("{resolved}");
Ok(())
}
fn resolve_calver_from_env() -> Result<String> {
resolve_calver(
&CalverEnv {
build_version: trim_option(env::var("BUILD_VERSION").ok()),
fluxer_build_version: trim_option(env::var("FLUXER_BUILD_VERSION").ok()),
fluxer_build_date: trim_option(env::var("FLUXER_BUILD_DATE").ok()),
},
Utc::now(),
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct CalverOutputs {
version: String,
time: String,
micro: String,
date: String,
year: String,
month: String,
day: String,
month_day: String,
}
fn calver_outputs(version: &str) -> Result<CalverOutputs> {
let instant = parse_version_instant(version)?;
let time = format!(
"{:02}{:02}{:02}",
instant.hour(),
instant.minute(),
instant.second()
);
let micro = time
.parse::<u32>()
.expect("HHMMSS time segment should parse")
.to_string();
Ok(CalverOutputs {
version: version.to_string(),
time,
micro,
date: format!(
"{:04}{:02}{:02}",
instant.year(),
instant.month(),
instant.day()
),
year: instant.year().to_string(),
month: instant.month().to_string(),
day: format!("{:02}", instant.day()),
month_day: format!("{}{:02}", instant.month(), instant.day()),
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::common::CalverEnv;
use chrono::{TimeZone, Utc};
#[test]
fn calver_outputs_match_legacy_shell_fields() {
assert_eq!(
calver_outputs("2026.520.10203").unwrap(),
CalverOutputs {
version: "2026.520.10203".to_string(),
time: "010203".to_string(),
micro: "10203".to_string(),
date: "20260520".to_string(),
year: "2026".to_string(),
month: "5".to_string(),
day: "20".to_string(),
month_day: "520".to_string(),
}
);
}
#[test]
fn calver_date_only_override_matches_legacy_shell() {
let version = resolve_calver(
&CalverEnv {
fluxer_build_date: Some("2026-01-09".to_string()),
..CalverEnv::default()
},
Utc.with_ymd_and_hms(2026, 5, 20, 1, 2, 3).single().unwrap(),
)
.unwrap();
assert_eq!(version, "2026.109.0");
}
#[test]
fn calver_rejects_invalid_time() {
assert_eq!(
calver_outputs("2026.520.246000").unwrap_err().to_string(),
"Invalid build version date/time: 2026.520.246000"
);
}
}
+248
View File
@@ -0,0 +1,248 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::{CommandSpec, run_command};
use crate::gateway::{GatewayStep, run_gateway_step};
use anyhow::{Context, Result};
use clap::{Args, ValueEnum};
use std::env;
use std::path::{Path, PathBuf};
#[derive(Debug, Args, Clone)]
pub struct CiArgs {
#[arg(long, value_enum)]
step: CiStep,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "snake_case")]
enum CiStep {
InstallDependencies,
Typecheck,
Test,
Knip,
GatewayFmt,
GatewayCompile,
GatewayDialyzer,
GatewayEunit,
}
#[derive(Debug, Args, Clone)]
pub struct CiScriptsArgs {
#[arg(long, value_enum)]
step: CiScriptsStep,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "snake_case")]
enum CiScriptsStep {
Sync,
Test,
}
pub async fn run_ci(args: CiArgs) -> Result<()> {
let root = repo_root()?;
match args.step {
CiStep::InstallDependencies => run_command(
CommandSpec::new("pnpm")
.args(["install", "--frozen-lockfile"])
.current_dir(root),
),
CiStep::Typecheck => {
run_generators(&root, true)?;
run_command(
CommandSpec::new("pnpm")
.args(["-r", "--if-present", "typecheck"])
.current_dir(root),
)
}
CiStep::Test => {
run_generators(&root, false)?;
run_workspace_tests(&root)?;
run_command(with_test_env(
CommandSpec::new("pnpm")
.args(["--filter", "fluxer_api", "test"])
.current_dir(root),
))
}
CiStep::Knip => {
run_command(
CommandSpec::new("pnpm")
.args(["--filter", "fluxer_app", "i18n:compile"])
.current_dir(&root),
)?;
run_command(
CommandSpec::new("pnpm")
.args(["exec", "knip"])
.current_dir(root),
)
}
CiStep::GatewayFmt => {
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::FmtCheck, "test")
}
CiStep::GatewayCompile => {
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Compile, "test")
}
CiStep::GatewayDialyzer => {
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Dialyzer, "test")
}
CiStep::GatewayEunit => {
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Eunit, "test")
}
}
}
pub async fn run_ci_scripts(args: CiScriptsArgs) -> Result<()> {
let root = repo_root()?;
match args.step {
CiScriptsStep::Sync => run_command(
CommandSpec::new("cargo")
.args([
"fetch",
"--locked",
"--manifest-path",
"tools/ci/Cargo.toml",
])
.current_dir(root),
),
CiScriptsStep::Test => run_command(
CommandSpec::new("cargo")
.args(["test", "--locked", "--manifest-path", "tools/ci/Cargo.toml"])
.current_dir(root),
),
}
}
fn run_generators(root: &Path, for_typecheck: bool) -> Result<()> {
for command in generator_commands(for_typecheck) {
run_command(command.current_dir(root))?;
}
Ok(())
}
fn generator_commands(for_typecheck: bool) -> Vec<CommandSpec> {
let mut commands = vec![
CommandSpec::new("pnpm").args(["--filter", "@fluxer/config", "generate"]),
CommandSpec::new("pnpm").args(["--filter", "@fluxer/schema", "generate"]),
];
if for_typecheck {
commands.push(CommandSpec::new("pnpm").args([
"--filter",
"@fluxer/i18n",
"generate:types",
]));
}
commands.push(CommandSpec::new("pnpm").args(["--filter", "fluxer_app", "i18n:compile"]));
commands
}
fn run_workspace_tests(root: &Path) -> Result<()> {
let workspace_concurrency =
env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").unwrap_or_else(|_| "2".to_string());
run_command(with_test_env(
CommandSpec::new("pnpm")
.args([
"-r",
&format!("--workspace-concurrency={workspace_concurrency}"),
"--filter",
"!fluxer_api",
"--filter",
"!fluxer",
"--if-present",
"test",
])
.current_dir(root),
))
}
fn with_test_env(spec: CommandSpec) -> CommandSpec {
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
let api_workers = env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| "2".to_string());
spec.env("FLUXER_NATS_URL", &nats_url)
.env(
"FLUXER_NATS_CORE_URL",
env::var("FLUXER_NATS_CORE_URL").unwrap_or_else(|_| nats_url.clone()),
)
.env(
"FLUXER_NATS_JETSTREAM_URL",
env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone()),
)
.env("API_TEST_MAX_WORKERS", &api_workers)
.env(
"API_TEST_MAX_CONCURRENCY",
env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers),
)
}
fn default_test_nats_url() -> String {
if Path::new("/.dockerenv").exists() {
"nats://nats:4222".to_string()
} else {
"nats://127.0.0.1:4222".to_string()
}
}
fn repo_root() -> Result<PathBuf> {
env::var("GITHUB_WORKSPACE")
.map(PathBuf::from)
.or_else(|_| env::current_dir())
.context("Failed to resolve repository root")
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::OsString;
#[test]
fn generator_commands_include_i18n_types_only_for_typecheck() {
let typecheck = generator_commands(true)
.into_iter()
.map(|command| command.args)
.collect::<Vec<_>>();
let test = generator_commands(false)
.into_iter()
.map(|command| command.args)
.collect::<Vec<_>>();
assert!(typecheck.contains(&vec![
OsString::from("--filter"),
OsString::from("@fluxer/i18n"),
OsString::from("generate:types"),
]));
assert!(!test.contains(&vec![
OsString::from("--filter"),
OsString::from("@fluxer/i18n"),
OsString::from("generate:types"),
]));
}
#[test]
fn with_test_env_sets_all_nats_urls_and_concurrency() {
let spec = with_test_env(CommandSpec::new("pnpm"));
let env = spec
.env
.into_iter()
.collect::<std::collections::BTreeMap<_, _>>();
assert_eq!(
env.get(&OsString::from("FLUXER_NATS_URL")),
Some(&OsString::from("nats://nats:4222"))
);
assert_eq!(
env.get(&OsString::from("FLUXER_NATS_CORE_URL")),
Some(&OsString::from("nats://nats:4222"))
);
assert_eq!(
env.get(&OsString::from("FLUXER_NATS_JETSTREAM_URL")),
Some(&OsString::from("nats://nats:4222"))
);
assert_eq!(
env.get(&OsString::from("API_TEST_MAX_WORKERS")),
Some(&OsString::from("2"))
);
assert_eq!(
env.get(&OsString::from("API_TEST_MAX_CONCURRENCY")),
Some(&OsString::from("2"))
);
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+496
View File
@@ -0,0 +1,496 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::{CommandSpec, command_succeeds, output_text, run_command};
use anyhow::{Context, Result, anyhow, bail, ensure};
use clap::{Args, ValueEnum};
use std::env;
use std::ffi::OsString;
use std::fs;
use std::path::{Path, PathBuf};
const GATEWAY_NIF_CRATES: &[&str] = &["push_markdown_plaintext_nif", "guild_member_list_oset_nif"];
#[derive(Debug, Args, Clone)]
pub struct BuildGatewayNifsArgs {
#[arg(long)]
gateway_dir: Option<PathBuf>,
}
#[derive(Debug, Args, Clone)]
pub struct GatewayArgs {
#[arg(long, value_enum)]
step: GatewayStep,
#[arg(long, default_value = "test")]
eqwalizer_profile: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[clap(rename_all = "snake_case")]
pub(crate) enum GatewayStep {
Fmt,
FmtCheck,
Lint,
Compile,
ProdCompile,
Dialyzer,
Eqwalizer,
Typecheck,
Eunit,
Bench,
AllChecks,
Clean,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum NifBuildProfile {
Debug,
Release,
}
impl NifBuildProfile {
fn from_env_value(value: Option<&str>) -> Self {
match value {
Some("release") | None => Self::Release,
Some(_) => Self::Debug,
}
}
fn target_dir_name(self) -> &'static str {
match self {
Self::Debug => "debug",
Self::Release => "release",
}
}
fn cargo_args(self) -> Vec<OsString> {
let mut args = vec![OsString::from("build")];
if self == Self::Release {
args.push(OsString::from("--release"));
}
args
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct GatewayNifBuild {
crate_name: String,
native_dir: PathBuf,
cargo_args: Vec<OsString>,
artifact_path: PathBuf,
output_path: PathBuf,
}
pub fn run_build_gateway_nifs(args: BuildGatewayNifsArgs) -> Result<()> {
let gateway_dir = args
.gateway_dir
.map(Ok)
.unwrap_or_else(resolve_gateway_dir)?;
build_gateway_nifs(&gateway_dir)
}
pub fn run_gateway(args: GatewayArgs) -> Result<()> {
let gateway_dir = resolve_gateway_dir()?;
run_gateway_step(&gateway_dir, args.step, &args.eqwalizer_profile)
}
pub(crate) fn run_gateway_step(
gateway_dir: &Path,
step: GatewayStep,
eqwalizer_profile: &str,
) -> Result<()> {
match step {
GatewayStep::Fmt => run_rebar(gateway_dir, ["fmt"]),
GatewayStep::FmtCheck => run_gateway_fmt_check(gateway_dir),
GatewayStep::Lint => run_rebar(gateway_dir, ["lint"]),
GatewayStep::Compile => run_rebar(gateway_dir, ["compile"]),
GatewayStep::ProdCompile => {
run_rebar(gateway_dir, ["as", "prod", "clean", "-a"])?;
run_rebar(gateway_dir, ["as", "prod", "compile"])
}
GatewayStep::Dialyzer => run_rebar(gateway_dir, ["dialyzer"]),
GatewayStep::Eqwalizer => run_eqwalizer(gateway_dir, eqwalizer_profile),
GatewayStep::Typecheck => run_eqwalizer(gateway_dir, eqwalizer_profile),
GatewayStep::Eunit => run_rebar(gateway_dir, ["as", "test", "eunit"]),
GatewayStep::Bench => run_rebar(gateway_dir, ["eunit", "--module=guild_member_list_bench"]),
GatewayStep::AllChecks => {
run_gateway_check_step("Step 1/5: Format check (erlfmt)", || {
run_gateway_step(gateway_dir, GatewayStep::FmtCheck, eqwalizer_profile)
})?;
run_gateway_check_step("Step 2/5: Lint (elvis)", || {
run_gateway_step(gateway_dir, GatewayStep::Lint, eqwalizer_profile)
})?;
run_gateway_check_step("Step 3/5: Compile", || {
run_gateway_step(gateway_dir, GatewayStep::Compile, eqwalizer_profile)
})?;
run_gateway_check_step("Step 4/5: Type check", || {
run_gateway_step(gateway_dir, GatewayStep::Typecheck, eqwalizer_profile)
})?;
run_gateway_check_step("Step 5/5: Unit tests (eunit)", || {
run_gateway_step(gateway_dir, GatewayStep::Eunit, eqwalizer_profile)
})?;
println!("All gateway checks passed.");
Ok(())
}
GatewayStep::Clean => {
run_rebar(gateway_dir, ["clean", "--all"])?;
let plugins_dir = gateway_dir.join("_build/default/plugins");
if plugins_dir.exists() {
fs::remove_dir_all(&plugins_dir)
.with_context(|| format!("Failed to remove {}", plugins_dir.display()))?;
}
println!("Cleaned gateway build outputs.");
Ok(())
}
}
}
fn build_gateway_nifs(gateway_dir: &Path) -> Result<()> {
let profile =
NifBuildProfile::from_env_value(env::var("FLUXER_GATEWAY_NIF_PROFILE").ok().as_deref());
let builds = gateway_nif_builds(
gateway_dir,
profile,
env::consts::DLL_PREFIX,
env::consts::DLL_EXTENSION,
);
let priv_dir = gateway_dir.join("priv");
fs::create_dir_all(&priv_dir)
.with_context(|| format!("Failed to create {}", priv_dir.display()))?;
for build in builds {
let mut cargo_args = build.cargo_args.clone();
cargo_args.push(OsString::from("--manifest-path"));
cargo_args.push(build.native_dir.join("Cargo.toml").into_os_string());
run_command(CommandSpec::new("cargo").args(cargo_args))?;
ensure!(
build.artifact_path.is_file(),
"Expected NIF artifact was not produced: {}",
build.artifact_path.display()
);
fs::copy(&build.artifact_path, &build.output_path).with_context(|| {
format!(
"Failed to copy {} to {}",
build.artifact_path.display(),
build.output_path.display()
)
})?;
println!(
"Installed gateway NIF {} -> {}",
build.crate_name,
build.output_path.display()
);
}
Ok(())
}
fn run_gateway_check_step(label: &str, run: impl FnOnce() -> Result<()>) -> Result<()> {
println!("========================================");
println!(" {label}");
println!("========================================");
run()?;
println!();
Ok(())
}
fn run_rebar(
gateway_dir: &Path,
args: impl IntoIterator<Item = impl Into<OsString>>,
) -> Result<()> {
run_command(rebar_command(gateway_dir, args))
}
fn run_gateway_fmt_check(gateway_dir: &Path) -> Result<()> {
ensure!(
command_succeeds(with_asdf_shims(CommandSpec::new("rebar3").arg("--version"))),
"rebar3 is required for gateway formatting"
);
let output = crate::common::capture(rebar_command(gateway_dir, ["fmt", "--check"]))?;
if output.status == 0 {
return Ok(());
}
let combined = format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
if combined.contains("Command fmt not found")
|| combined.to_ascii_lowercase().contains("not found")
{
println!("rebar3 fmt plugin is not configured; skipping gateway formatting check.");
return Ok(());
}
bail!("gateway formatting failed with exit code {}", output.status)
}
fn run_eqwalizer(gateway_dir: &Path, profile: &str) -> Result<()> {
ensure!(
!profile.is_empty(),
"--eqwalizer-profile requires a non-empty profile name"
);
ensure!(
command_succeeds(with_asdf_shims(CommandSpec::new("elp").arg("version"))),
"elp not found in PATH. Install ELP from https://github.com/WhatsApp/erlang-language-platform/releases"
);
ensure!(
command_succeeds(with_asdf_shims(CommandSpec::new("erl").arg("-version"))),
"erl not found in PATH"
);
let erlang_source = find_erlang_source()?;
let elp_version = output_text(with_asdf_shims(CommandSpec::new("elp").arg("version")))?;
println!("==> Running eqWAlizer with {elp_version}");
println!("==> Using Erlang source: {erlang_source}");
println!("==> Rebar profile: {profile}");
run_command(
with_asdf_shims(
CommandSpec::new("elp")
.args([
"eqwalize-all",
"--rebar",
"--as",
profile,
"--stats",
"--bail-on-error",
])
.current_dir(gateway_dir),
)
.env("REBAR_SKIP_PROJECT_PLUGINS", "1"),
)
}
fn find_erlang_source() -> Result<String> {
output_text(with_asdf_shims(CommandSpec::new("erl").args([
"-noshell",
"-eval",
concat!(
"Root = code:root_dir(), ",
"Matches = filelib:wildcard(filename:join([Root, \"lib\", \"erts-*\", \"src\", \"erlang.erl\"])), ",
"case Matches of ",
"[Path | _] -> io:format(\"~s~n\", [Path]), halt(0); ",
"[] -> halt(2) ",
"end."
),
])))
.context(
"Erlang/OTP source files are required for Eqwalizer. On Debian/Ubuntu, install erlang-src",
)
}
fn rebar_command(
gateway_dir: &Path,
args: impl IntoIterator<Item = impl Into<OsString>>,
) -> CommandSpec {
let args = args.into_iter().map(Into::into).collect::<Vec<_>>();
let should_skip_plugins = should_skip_rebar_project_plugins(&args);
let mut spec = with_asdf_shims(
CommandSpec::new("rebar3")
.args(args)
.current_dir(gateway_dir),
);
if should_skip_plugins {
spec = spec.env("REBAR_SKIP_PROJECT_PLUGINS", "1");
}
spec
}
fn should_skip_rebar_project_plugins(args: &[OsString]) -> bool {
!args
.iter()
.any(|arg| matches!(arg.to_string_lossy().as_ref(), "fmt" | "lint" | "plugins"))
}
fn with_asdf_shims(spec: CommandSpec) -> CommandSpec {
let Some(shims_path) = asdf_shims_path() else {
return spec;
};
if !shims_path.is_dir() {
return spec;
}
let path = env::var_os("PATH").unwrap_or_default();
let mut paths = std::iter::once(shims_path)
.chain(env::split_paths(&path))
.collect::<Vec<_>>();
let joined = env::join_paths(paths.drain(..)).unwrap_or(path);
spec.env("PATH", joined)
}
fn asdf_shims_path() -> Option<PathBuf> {
if let Some(asdf_data_dir) = env::var_os("ASDF_DATA_DIR") {
return Some(PathBuf::from(asdf_data_dir).join("shims"));
}
env::var_os("HOME")
.filter(|home| !home.is_empty())
.map(|home| PathBuf::from(home).join(".asdf/shims"))
}
fn gateway_nif_builds(
gateway_dir: &Path,
profile: NifBuildProfile,
dll_prefix: &str,
dll_extension: &str,
) -> Vec<GatewayNifBuild> {
GATEWAY_NIF_CRATES
.iter()
.map(|crate_name| {
gateway_nif_build(gateway_dir, profile, dll_prefix, dll_extension, crate_name)
})
.collect()
}
fn gateway_nif_build(
gateway_dir: &Path,
profile: NifBuildProfile,
dll_prefix: &str,
dll_extension: &str,
crate_name: &str,
) -> GatewayNifBuild {
let native_dir = gateway_dir.join("native").join(crate_name);
GatewayNifBuild {
crate_name: crate_name.to_string(),
cargo_args: profile.cargo_args(),
artifact_path: native_dir
.join("target")
.join(profile.target_dir_name())
.join(format!("{dll_prefix}{crate_name}.{dll_extension}")),
output_path: gateway_dir.join("priv").join(format!("{crate_name}.so")),
native_dir,
}
}
fn resolve_gateway_dir() -> Result<PathBuf> {
let cwd = env::current_dir().context("Failed to resolve current directory")?;
if cwd.join("rebar.config").is_file()
&& cwd.file_name().and_then(|value| value.to_str()) == Some("fluxer_gateway")
{
return Ok(cwd);
}
if cwd.join("fluxer_gateway/rebar.config").is_file() {
return Ok(cwd.join("fluxer_gateway"));
}
Err(anyhow!(
"Could not resolve fluxer_gateway directory from {}",
cwd.display()
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn nif_profile_defaults_to_release_and_uses_debug_for_other_values() {
assert_eq!(
NifBuildProfile::from_env_value(None),
NifBuildProfile::Release
);
assert_eq!(
NifBuildProfile::from_env_value(Some("release")),
NifBuildProfile::Release
);
assert_eq!(
NifBuildProfile::from_env_value(Some("debug")),
NifBuildProfile::Debug
);
assert_eq!(
NifBuildProfile::from_env_value(Some("dev")),
NifBuildProfile::Debug
);
}
#[test]
fn gateway_nif_build_plan_matches_legacy_artifact_layout() {
let gateway_dir = Path::new("/repo/fluxer_gateway");
let builds = gateway_nif_builds(gateway_dir, NifBuildProfile::Release, "lib", "so");
assert_eq!(builds.len(), 2);
assert_eq!(
builds[0],
GatewayNifBuild {
crate_name: "push_markdown_plaintext_nif".to_string(),
native_dir: PathBuf::from(
"/repo/fluxer_gateway/native/push_markdown_plaintext_nif"
),
cargo_args: vec![OsString::from("build"), OsString::from("--release")],
artifact_path: PathBuf::from(
"/repo/fluxer_gateway/native/push_markdown_plaintext_nif/target/release/libpush_markdown_plaintext_nif.so"
),
output_path: PathBuf::from(
"/repo/fluxer_gateway/priv/push_markdown_plaintext_nif.so"
),
}
);
}
#[test]
fn debug_build_plan_omits_release_arg_and_uses_debug_target_dir() {
let build = gateway_nif_build(
Path::new("/repo/fluxer_gateway"),
NifBuildProfile::Debug,
"lib",
"dylib",
"guild_member_list_oset_nif",
);
assert_eq!(build.cargo_args, vec![OsString::from("build")]);
assert_eq!(
build.artifact_path,
PathBuf::from(
"/repo/fluxer_gateway/native/guild_member_list_oset_nif/target/debug/libguild_member_list_oset_nif.dylib"
)
);
assert_eq!(
build.output_path,
PathBuf::from("/repo/fluxer_gateway/priv/guild_member_list_oset_nif.so")
);
}
#[test]
fn rebar_project_plugins_are_skipped_except_for_plugin_commands() {
assert!(should_skip_rebar_project_plugins(&[OsString::from(
"compile"
)]));
assert!(should_skip_rebar_project_plugins(&[
OsString::from("as"),
OsString::from("test"),
OsString::from("eunit"),
]));
assert!(!should_skip_rebar_project_plugins(&[OsString::from("fmt")]));
assert!(!should_skip_rebar_project_plugins(&[OsString::from(
"lint"
)]));
assert!(!should_skip_rebar_project_plugins(&[OsString::from(
"plugins"
)]));
}
#[test]
fn rebar_command_runs_in_gateway_dir_and_sets_skip_env_for_compile() {
let command = rebar_command(Path::new("/repo/fluxer_gateway"), ["compile"]);
assert_eq!(command.program, OsString::from("rebar3"));
assert_eq!(command.args, vec![OsString::from("compile")]);
assert_eq!(command.cwd, Some(PathBuf::from("/repo/fluxer_gateway")));
assert!(command.env.contains(&(
OsString::from("REBAR_SKIP_PROJECT_PLUGINS"),
OsString::from("1")
)));
}
#[test]
fn rebar_command_keeps_project_plugins_for_fmt() {
let command = rebar_command(Path::new("/repo/fluxer_gateway"), ["fmt", "--check"]);
assert_eq!(
command.args,
vec![OsString::from("fmt"), OsString::from("--check")]
);
assert!(
!command
.env
.iter()
.any(|(key, _)| key == &OsString::from("REBAR_SKIP_PROJECT_PLUGINS"))
);
}
}
+67
View File
@@ -0,0 +1,67 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
mod app_dev_server;
mod app_proxy;
mod app_wasm;
mod calver;
mod ci_workflow;
mod common;
mod desktop;
mod desktop_native;
mod gateway;
mod schema;
mod static_bucket;
use anyhow::Result;
use clap::{Parser, Subcommand};
#[derive(Debug, Parser)]
#[command(name = "fluxer-ci")]
struct Cli {
#[command(subcommand)]
command: Command,
}
#[derive(Debug, Subcommand)]
#[allow(clippy::large_enum_variant)]
enum Command {
AppDevServer(app_dev_server::AppDevServerArgs),
BuildAppWasm(app_wasm::BuildAppWasmArgs),
BuildAppProxy(app_proxy::BuildAppProxyArgs),
BuildMarkdownParserWasm(app_wasm::BuildMarkdownParserWasmArgs),
BuildDesktop(desktop::BuildDesktopArgs),
BuildDesktopNativeAddon(desktop_native::BuildDesktopNativeAddonArgs),
BuildGatewayNifs(gateway::BuildGatewayNifsArgs),
Ci(ci_workflow::CiArgs),
CiScripts(ci_workflow::CiScriptsArgs),
CleanSchemaGeneratedFiles(schema::CleanSchemaGeneratedFilesArgs),
Gateway(gateway::GatewayArgs),
RepairStaticAssetMetadata(static_bucket::RepairStaticAssetMetadataArgs),
ResolveCalver(calver::ResolveCalverArgs),
SyncStaticBucket(static_bucket::SyncStaticBucketArgs),
TestWebrtcSenderRust(desktop_native::TestWebrtcSenderRustArgs),
}
pub async fn run() -> Result<()> {
match Cli::parse().command {
Command::AppDevServer(args) => app_dev_server::run(args).await,
Command::BuildAppWasm(args) => app_wasm::run_build_app_wasm(args),
Command::BuildAppProxy(args) => app_proxy::run(args).await,
Command::BuildMarkdownParserWasm(args) => app_wasm::run_build_markdown_parser_wasm(args),
Command::BuildDesktop(args) => desktop::run(args).await,
Command::BuildDesktopNativeAddon(args) => {
desktop_native::run_build_desktop_native_addon(args)
}
Command::BuildGatewayNifs(args) => gateway::run_build_gateway_nifs(args),
Command::Ci(args) => ci_workflow::run_ci(args).await,
Command::CiScripts(args) => ci_workflow::run_ci_scripts(args).await,
Command::CleanSchemaGeneratedFiles(args) => schema::run_clean_generated_files(args),
Command::Gateway(args) => gateway::run_gateway(args),
Command::RepairStaticAssetMetadata(args) => {
static_bucket::repair_asset_metadata(args).await
}
Command::ResolveCalver(args) => calver::run(args),
Command::SyncStaticBucket(args) => static_bucket::run(args).await,
Command::TestWebrtcSenderRust(args) => desktop_native::run_test_webrtc_sender_rust(args),
}
}
+9
View File
@@ -0,0 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#[tokio::main]
async fn main() {
if let Err(error) = fluxer_ci::run().await {
eprintln!("{error:?}");
std::process::exit(1);
}
}
+115
View File
@@ -0,0 +1,115 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::collect_files;
use anyhow::{Context, Result};
use clap::Args;
use std::fs;
use std::path::{Path, PathBuf};
#[derive(Debug, Args, Clone)]
pub struct CleanSchemaGeneratedFilesArgs {
#[arg(long, default_value = "packages/schema/src/gen")]
root: PathBuf,
}
pub fn run_clean_generated_files(args: CleanSchemaGeneratedFilesArgs) -> Result<()> {
clean_generated_files(&args.root)
}
fn clean_generated_files(root: &Path) -> Result<()> {
for file in collect_files(root)?
.into_iter()
.filter(|path| path.extension().and_then(|value| value.to_str()) == Some("ts"))
{
let source = fs::read_to_string(&file)
.with_context(|| format!("Failed to read {}", file.display()))?;
let Some(import_index) = find_import_start(&source) else {
continue;
};
let content = format!(
"{}\n",
collapse_extra_blank_lines(source[import_index..].trim_end())
);
if content != source {
fs::write(&file, content)
.with_context(|| format!("Failed to write {}", file.display()))?;
}
}
Ok(())
}
fn find_import_start(source: &str) -> Option<usize> {
let mut offset = 0usize;
for line in source.split_inclusive('\n') {
if line.starts_with("import ") {
return Some(offset);
}
offset += line.len();
}
if source[offset..].starts_with("import ") {
return Some(offset);
}
None
}
fn collapse_extra_blank_lines(source: &str) -> String {
let mut output = String::with_capacity(source.len());
let mut consecutive_newlines = 0usize;
for ch in source.chars() {
if ch == '\n' {
consecutive_newlines += 1;
if consecutive_newlines <= 2 {
output.push(ch);
}
} else {
consecutive_newlines = 0;
output.push(ch);
}
}
output
}
#[cfg(test)]
mod tests {
use super::*;
fn write_file(path: &Path, contents: &str) {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(path, contents).unwrap();
}
#[test]
fn find_import_start_only_matches_line_starts() {
assert_eq!(
find_import_start("// import nope\nimport {x} from 'x';\n"),
Some(15)
);
assert_eq!(find_import_start("const value = 'import nope';\n"), None);
}
#[test]
fn collapse_extra_blank_lines_keeps_at_most_one_blank_line() {
assert_eq!(collapse_extra_blank_lines("a\n\n\n\nb\n\nc"), "a\n\nb\n\nc");
}
#[test]
fn clean_generated_files_removes_prelude_and_compacts_blank_lines() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path();
let file = root.join("generated.ts");
write_file(
&file,
"// header\n\n\nimport {x} from 'x';\n\n\n\nexport const y = x;\n\n",
);
write_file(&root.join("keep.txt"), "// header\n");
clean_generated_files(root).unwrap();
assert_eq!(
fs::read_to_string(file).unwrap(),
"import {x} from 'x';\n\nexport const y = x;\n"
);
}
}
+266
View File
@@ -0,0 +1,266 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::common::{
S3UploadPlanItem, collect_files, delete_s3_objects, list_s3_keys, path_to_s3_key,
replace_s3_object_metadata, s3_client, s3_content_type_for_key, upload_s3_plan_sync,
};
use anyhow::{Context, Result, ensure};
use clap::Args;
use std::collections::BTreeMap;
use std::env;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::sync::Semaphore;
use tokio::task::JoinSet;
const DEFAULT_SOURCE: &str = "fluxer_static";
const DEFAULT_STATIC_BUCKET: &str = "fluxer-static";
const DEFAULT_S3_ENDPOINT: &str = "https://ewr1.vultrobjects.com";
const DEFAULT_ASSET_PREFIX: &str = "assets/";
const IMMUTABLE_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable";
const DEFAULT_REPAIR_CONCURRENCY: usize = 8;
#[derive(Debug, Args, Clone)]
pub struct SyncStaticBucketArgs {
#[arg(long, default_value = DEFAULT_SOURCE)]
source: PathBuf,
#[arg(long, default_value = DEFAULT_STATIC_BUCKET)]
bucket: String,
}
#[derive(Debug, Args, Clone)]
pub struct RepairStaticAssetMetadataArgs {
#[arg(long, default_value = DEFAULT_STATIC_BUCKET)]
bucket: String,
#[arg(long, default_value = DEFAULT_ASSET_PREFIX)]
prefix: String,
}
pub async fn run(args: SyncStaticBucketArgs) -> Result<()> {
ensure!(
args.source.is_dir(),
"Static source directory is missing: {}",
args.source.display()
);
let upload_plan = static_upload_plan(&args.source)?;
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
let plan = upload_plan
.into_iter()
.map(|(key, path)| S3UploadPlanItem::new(path, key).with_detected_content_type())
.collect::<Vec<_>>();
let stats = upload_s3_plan_sync(&client, &args.bucket, plan).await?;
println!(
"Static bucket sync complete: uploaded {} file(s), skipped existing {}",
stats.uploaded, stats.skipped_existing
);
let remote_keys = list_s3_keys(&client, &args.bucket, "").await?;
let markdown_keys = remote_keys
.into_iter()
.filter(|key| key.to_ascii_lowercase().ends_with(".md"))
.collect::<Vec<_>>();
let removed = delete_s3_objects(&client, &args.bucket, &markdown_keys).await?;
println!("Static bucket sync removed {removed} stray .md object(s)");
Ok(())
}
pub async fn repair_asset_metadata(args: RepairStaticAssetMetadataArgs) -> Result<()> {
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
let keys = list_s3_keys(&client, &args.bucket, &args.prefix).await?;
let mut skipped = 0_usize;
let mut tasks = JoinSet::new();
let semaphore = Arc::new(Semaphore::new(static_asset_repair_concurrency()));
for key in keys {
let Some(content_type) = s3_content_type_for_key(&key) else {
skipped += 1;
continue;
};
let permit = semaphore
.clone()
.acquire_owned()
.await
.context("S3 repair semaphore closed")?;
let client = client.clone();
let bucket = args.bucket.clone();
tasks.spawn(async move {
let _permit = permit;
replace_s3_object_metadata(
&client,
&bucket,
&key,
Some(content_type),
Some(IMMUTABLE_ASSET_CACHE_CONTROL),
)
.await?;
Ok::<_, anyhow::Error>(())
});
}
let mut repaired = 0_usize;
while let Some(result) = tasks.join_next().await {
result.context("S3 metadata repair task failed")??;
repaired += 1;
}
println!(
"Static asset metadata repair complete: repaired {repaired} file(s), skipped {skipped}"
);
Ok(())
}
fn static_asset_repair_concurrency() -> usize {
env::var("S3_WRITE_CONCURRENCY")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_REPAIR_CONCURRENCY)
}
fn static_upload_plan(source: &Path) -> Result<BTreeMap<String, PathBuf>> {
let mut plan = BTreeMap::new();
for file in collect_files(source)? {
let relative = file
.strip_prefix(source)
.with_context(|| format!("Failed to relativize {}", file.display()))?;
if !should_sync_static_path(relative) {
continue;
}
let key = path_to_s3_key(relative);
if !key.is_empty() {
plan.insert(key, file);
}
}
Ok(plan)
}
fn should_sync_static_path(relative: &Path) -> bool {
if relative
.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
{
return false;
}
let Some(first) = relative.components().next() else {
return false;
};
match first {
std::path::Component::Normal(value) => {
let value = value.to_string_lossy();
value != ".github" && value != "assets"
}
_ => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
fn write_file(path: &Path, contents: &str) {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(path, contents).unwrap();
}
#[test]
fn upload_plan_excludes_github_and_assets_roots() {
let temp = tempfile::tempdir().unwrap();
let source = temp.path();
write_file(&source.join("index.html"), "html");
write_file(&source.join("docs/install.html"), "docs");
write_file(&source.join(".github/workflows/ignored.yaml"), "workflow");
write_file(&source.join("assets/app.js"), "asset");
let keys = static_upload_plan(source)
.unwrap()
.keys()
.cloned()
.collect::<Vec<_>>();
assert_eq!(keys, vec!["docs/install.html", "index.html"]);
}
#[test]
fn upload_plan_is_deterministic_and_keeps_non_root_assets_paths() {
let temp = tempfile::tempdir().unwrap();
let source = temp.path();
write_file(&source.join("z.html"), "z");
write_file(&source.join("docs/assets/keep.js"), "keep");
write_file(&source.join("a.html"), "a");
let keys = static_upload_plan(source)
.unwrap()
.keys()
.cloned()
.collect::<Vec<_>>();
assert_eq!(keys, vec!["a.html", "docs/assets/keep.js", "z.html"]);
}
#[test]
fn static_key_filter_matches_workflow_excludes() {
assert!(should_sync_static_path(Path::new("index.html")));
assert!(should_sync_static_path(
Path::new("docs").join("install.html").as_path()
));
assert!(!should_sync_static_path(
Path::new("assets").join("app.js").as_path()
));
assert!(!should_sync_static_path(
Path::new(".github")
.join("workflows")
.join("sync.yaml")
.as_path()
));
assert!(!should_sync_static_path(Path::new("")));
}
#[test]
fn static_key_filter_excludes_markdown() {
assert!(!should_sync_static_path(Path::new(
"THIRD_PARTY_LICENSES.md"
)));
assert!(!should_sync_static_path(
Path::new("fonts").join("NOTICE.md").as_path()
));
assert!(!should_sync_static_path(
Path::new("emoji").join("README.MD").as_path()
));
assert!(should_sync_static_path(Path::new("index.html")));
}
#[test]
fn upload_plan_excludes_markdown_files() {
let temp = tempfile::tempdir().unwrap();
let source = temp.path();
write_file(&source.join("index.html"), "html");
write_file(&source.join("THIRD_PARTY_LICENSES.md"), "licenses");
write_file(&source.join("fonts/NOTICE.md"), "notice");
let keys = static_upload_plan(source)
.unwrap()
.keys()
.cloned()
.collect::<Vec<_>>();
assert_eq!(keys, vec!["index.html"]);
}
#[test]
fn repair_metadata_args_default_to_app_assets_prefix() {
let args = RepairStaticAssetMetadataArgs {
bucket: DEFAULT_STATIC_BUCKET.to_string(),
prefix: DEFAULT_ASSET_PREFIX.to_string(),
};
assert_eq!(args.bucket, "fluxer-static");
assert_eq!(args.prefix, "assets/");
}
}
+79
View File
@@ -0,0 +1,79 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export interface RgbaTransformResult {
rgba: Uint8Array;
width: number;
height: number;
}
export interface DecodedFrame {
rgba: Uint8Array;
width: number;
height: number;
delayMs: number;
}
export interface EncodedApngFrame {
compressed: Uint8Array;
width: number;
height: number;
delayMs: number;
}
export interface EncodedGifChunk {
data: Uint8Array;
width: number;
height: number;
}
export function assemble_apng_frames(frames: Array<EncodedApngFrame>): Uint8Array;
export function assemble_gif_frame_chunks(chunks: Array<EncodedGifChunk>): Uint8Array;
export function crop_and_rotate_apng(
input: Uint8Array,
x: number,
y: number,
width: number,
height: number,
rotation_deg: number,
resize_width?: number | null,
resize_height?: number | null,
): Uint8Array;
export function crop_and_rotate_gif(
input: Uint8Array,
x: number,
y: number,
width: number,
height: number,
rotation_deg: number,
resize_width?: number | null,
resize_height?: number | null,
): Uint8Array;
export function crop_and_rotate_image(
input: Uint8Array,
format_hint: string,
x: number,
y: number,
width: number,
height: number,
rotation_deg: number,
resize_width?: number | null,
resize_height?: number | null,
): Uint8Array;
export function crop_rotate_rgba(
input: Uint8Array,
src_width: number,
src_height: number,
x: number,
y: number,
width: number,
height: number,
rotation_deg: number,
resize_width?: number | null,
resize_height?: number | null,
): RgbaTransformResult;
export function decode_apng_frames(input: Uint8Array): Array<DecodedFrame>;
export function decode_gif_frames(input: Uint8Array): Array<DecodedFrame>;
export function encode_apng_frame_payload(frame: DecodedFrame): EncodedApngFrame;
export function encode_apng_frames(frames: Array<DecodedFrame>): Uint8Array;
export function encode_gif_frame_chunk(frame: DecodedFrame, first?: boolean): EncodedGifChunk;
export function encode_gif_frames(frames: Array<DecodedFrame>): Uint8Array;
+757
View File
@@ -0,0 +1,757 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {encode as encodePng} from 'fast-png';
import {zlibSync} from 'fflate';
import {applyPalette, GIFEncoder, quantize} from 'gifenc';
import {decompressFrames, parseGIF} from 'gifuct-js';
import {decode as decodeJpeg, encode as encodeJpeg} from 'jpeg-js';
import UPNG from 'upng-js';
const textEncoder = new TextEncoder();
const NULL_U32 = 0xffffffff;
const RGBA_RESULT_HEADER_BYTES = 8;
const MAX_ANIMATION_PIXELS = 200_000_000;
const MAX_STATIC_DECODE_MEMORY_MB = 1024;
function inputBytes(input) {
if (input == null) return new Uint8Array();
return input instanceof Uint8Array ? input : new Uint8Array(input);
}
function arrayBufferFor(bytes) {
if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) return bytes.buffer;
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
}
function nonNegativeU32(value) {
const number = Number(value);
if (!Number.isFinite(number) || number <= 0) return 0;
return Math.min(NULL_U32, Math.floor(number));
}
function cropCoordU32(value) {
const number = Number(value);
if (!Number.isFinite(number) || number <= 0) return 0;
return Math.min(NULL_U32, Math.floor(number));
}
function optionalDimension(value) {
const dimension = nonNegativeU32(value);
return dimension > 0 ? dimension : null;
}
function effectiveRotation(rotationDeg) {
const rotation = ((Math.floor(Number(rotationDeg) || 0) % 360) + 360) % 360;
return rotation === 90 || rotation === 180 || rotation === 270 ? rotation : 0;
}
function normalizedFormat(value) {
const format = String(value ?? '')
.trim()
.toLowerCase();
if (format === 'jpg') return 'jpeg';
if (format === 'apng') return 'png';
if (format === 'animated_webp') return 'webp';
return format;
}
function sniffImageFormat(input) {
const bytes = inputBytes(input);
if (
bytes.length >= 8 &&
bytes[0] === 0x89 &&
bytes[1] === 0x50 &&
bytes[2] === 0x4e &&
bytes[3] === 0x47 &&
bytes[4] === 0x0d &&
bytes[5] === 0x0a &&
bytes[6] === 0x1a &&
bytes[7] === 0x0a
) {
return 'png';
}
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'jpeg';
if (
bytes.length >= 6 &&
bytes[0] === 0x47 &&
bytes[1] === 0x49 &&
bytes[2] === 0x46 &&
bytes[3] === 0x38 &&
(bytes[4] === 0x37 || bytes[4] === 0x39) &&
bytes[5] === 0x61
) {
return 'gif';
}
if (
bytes.length >= 12 &&
bytes[0] === 0x52 &&
bytes[1] === 0x49 &&
bytes[2] === 0x46 &&
bytes[3] === 0x46 &&
bytes[8] === 0x57 &&
bytes[9] === 0x45 &&
bytes[10] === 0x42 &&
bytes[11] === 0x50
) {
return 'webp';
}
if (
bytes.length >= 12 &&
bytes[4] === 0x66 &&
bytes[5] === 0x74 &&
bytes[6] === 0x79 &&
bytes[7] === 0x70 &&
bytes[8] === 0x61 &&
bytes[9] === 0x76 &&
bytes[10] === 0x69 &&
(bytes[11] === 0x66 || bytes[11] === 0x73)
) {
return 'avif';
}
return 'unknown';
}
function readU32FromBytes(bytes, offset) {
return new DataView(bytes.buffer, bytes.byteOffset + offset, 4).getUint32(0, true);
}
function readU16LE(bytes, offset) {
return bytes[offset] | (bytes[offset + 1] << 8);
}
function readU16BE(bytes, offset) {
return (bytes[offset] << 8) | bytes[offset + 1];
}
function readU32BE(bytes, offset) {
return (bytes[offset] * 0x1000000 + ((bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3])) >>> 0;
}
function parseRgbaTransformResult(bytes) {
if (bytes.byteLength < RGBA_RESULT_HEADER_BYTES) throw new Error('libfluxcore returned a truncated RGBA result');
const width = readU32FromBytes(bytes, 0);
const height = readU32FromBytes(bytes, 4);
const expected = RGBA_RESULT_HEADER_BYTES + width * height * 4;
if (bytes.byteLength !== expected) throw new Error('libfluxcore returned an invalid RGBA result length');
return {rgba: bytes.subarray(RGBA_RESULT_HEADER_BYTES), width, height};
}
function pngDimensions(bytes) {
if (bytes.length < 24 || sniffImageFormat(bytes) !== 'png') return null;
if (bytes[12] !== 0x49 || bytes[13] !== 0x48 || bytes[14] !== 0x44 || bytes[15] !== 0x52) return null;
return {width: readU32BE(bytes, 16), height: readU32BE(bytes, 20)};
}
function gifDimensions(bytes) {
if (bytes.length < 10 || sniffImageFormat(bytes) !== 'gif') return null;
return {width: readU16LE(bytes, 6), height: readU16LE(bytes, 8)};
}
function jpegDimensions(bytes) {
if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null;
let offset = 2;
while (offset + 4 <= bytes.length) {
while (offset < bytes.length && bytes[offset] === 0xff) offset += 1;
if (offset >= bytes.length) return null;
const marker = bytes[offset];
offset += 1;
if (marker === 0xd9 || marker === 0xda) return null;
if (offset + 2 > bytes.length) return null;
const length = readU16BE(bytes, offset);
if (length < 2 || offset + length > bytes.length) return null;
if (
marker === 0xc0 ||
marker === 0xc1 ||
marker === 0xc2 ||
marker === 0xc3 ||
marker === 0xc5 ||
marker === 0xc6 ||
marker === 0xc7 ||
marker === 0xc9 ||
marker === 0xca ||
marker === 0xcb ||
marker === 0xcd ||
marker === 0xce ||
marker === 0xcf
) {
if (length < 7) return null;
return {height: readU16BE(bytes, offset + 3), width: readU16BE(bytes, offset + 5)};
}
offset += length;
}
return null;
}
function imageDimensions(bytes, format) {
switch (format) {
case 'png':
return pngDimensions(bytes);
case 'gif':
return gifDimensions(bytes);
case 'jpeg':
return jpegDimensions(bytes);
default:
return null;
}
}
function isNoopTransform(imageWidth, imageHeight, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
const cropX = Math.min(cropCoordU32(x), imageWidth);
const cropY = Math.min(cropCoordU32(y), imageHeight);
const cropW = Math.min(nonNegativeU32(width), imageWidth - cropX);
const cropH = Math.min(nonNegativeU32(height), imageHeight - cropY);
const targetW = optionalDimension(resizeWidth) ?? imageWidth;
const targetH = optionalDimension(resizeHeight) ?? imageHeight;
return (
cropX === 0 &&
cropY === 0 &&
cropW === imageWidth &&
cropH === imageHeight &&
effectiveRotation(rotationDeg) === 0 &&
targetW === imageWidth &&
targetH === imageHeight
);
}
export function crop_rotate_rgba(
input,
src_width,
src_height,
x,
y,
width,
height,
rotation_deg,
resize_width,
resize_height,
) {
const sourceWidth = nonNegativeU32(src_width);
const sourceHeight = nonNegativeU32(src_height);
return parseRgbaTransformResult(
// biome-ignore lint/correctness/noUndeclaredVariables: Injected by the generated wasm wrapper at build time.
crop_rotate_rgba_raw(
inputBytes(input),
sourceWidth,
sourceHeight,
cropCoordU32(x),
cropCoordU32(y),
nonNegativeU32(width),
nonNegativeU32(height),
effectiveRotation(rotation_deg),
optionalDimension(resize_width),
optionalDimension(resize_height),
),
);
}
function transformFrame(frame, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
const transformed = crop_rotate_rgba(
frame.rgba,
frame.width,
frame.height,
x,
y,
width,
height,
rotationDeg,
resizeWidth,
resizeHeight,
);
return {rgba: transformed.rgba, width: transformed.width, height: transformed.height, delayMs: frame.delayMs};
}
function transformFrames(frames, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
let totalPixels = 0;
const transformed = frames.map((frame) => {
const out = transformFrame(frame, x, y, width, height, rotationDeg, resizeWidth, resizeHeight);
totalPixels += out.width * out.height;
if (totalPixels > MAX_ANIMATION_PIXELS) {
throw new Error('Animated image is too large to crop. Try reducing its dimensions or number of frames.');
}
return out;
});
return transformed;
}
function decodePngFrames(input) {
const bytes = inputBytes(input);
const decoded = UPNG.decode(arrayBufferFor(bytes));
const rgbaFrames = UPNG.toRGBA8(decoded);
if (!rgbaFrames.length) throw new Error('PNG has no frames');
return rgbaFrames.map((frame, index) => {
const delayMs = decoded.frames?.[index]?.delay ?? 0;
return {rgba: new Uint8Array(frame), width: decoded.width, height: decoded.height, delayMs};
});
}
function decodeJpegFrame(input) {
const decoded = decodeJpeg(inputBytes(input), {
useTArray: true,
formatAsRGBA: true,
maxMemoryUsageInMB: MAX_STATIC_DECODE_MEMORY_MB,
});
return {rgba: new Uint8Array(decoded.data), width: decoded.width, height: decoded.height, delayMs: 0};
}
function drawGifPatch(canvas, canvasWidth, frame) {
const dims = frame.dims;
const patch = frame.patch;
for (let row = 0; row < dims.height; row += 1) {
const canvasY = dims.top + row;
if (canvasY < 0) continue;
const canvasOffset = (canvasY * canvasWidth + dims.left) * 4;
const patchOffset = row * dims.width * 4;
if (canvasOffset < 0 || canvasOffset >= canvas.length) continue;
for (let col = 0; col < dims.width; col += 1) {
const source = patchOffset + col * 4;
const target = canvasOffset + col * 4;
if (target < 0 || target + 4 > canvas.length || source + 4 > patch.length) continue;
if (patch[source + 3] === 0) continue;
canvas[target] = patch[source];
canvas[target + 1] = patch[source + 1];
canvas[target + 2] = patch[source + 2];
canvas[target + 3] = patch[source + 3];
}
}
}
function clearRectRgba(canvas, canvasWidth, x, y, width, height) {
for (let row = 0; row < height; row += 1) {
const start = ((y + row) * canvasWidth + x) * 4;
const end = start + width * 4;
if (start >= 0 && end <= canvas.length) canvas.fill(0, start, end);
}
}
function collectGifFrames(input, transformOptions) {
const bytes = inputBytes(input);
const parsed = parseGIF(arrayBufferFor(bytes));
const screenWidth = parsed.lsd.width;
const screenHeight = parsed.lsd.height;
const decodedFrames = decompressFrames(parsed, true);
if (!decodedFrames.length) throw new Error('GIF has no frames');
const canvas = new Uint8Array(screenWidth * screenHeight * 4);
let previousCanvas = null;
const frames = [];
for (const frame of decodedFrames) {
if (frame.disposalType === 3) previousCanvas = canvas.slice();
drawGifPatch(canvas, screenWidth, frame);
const sourceFrame = {
rgba: transformOptions ? canvas : canvas.slice(),
width: screenWidth,
height: screenHeight,
delayMs: frame.delay || 100,
};
frames.push(
transformOptions
? transformFrame(
sourceFrame,
transformOptions.x,
transformOptions.y,
transformOptions.width,
transformOptions.height,
transformOptions.rotationDeg,
transformOptions.resizeWidth,
transformOptions.resizeHeight,
)
: sourceFrame,
);
if (frame.disposalType === 2) {
clearRectRgba(canvas, screenWidth, frame.dims.left, frame.dims.top, frame.dims.width, frame.dims.height);
} else if (frame.disposalType === 3 && previousCanvas) {
canvas.set(previousCanvas);
previousCanvas = null;
}
}
return frames;
}
function decodeGifFrames(input) {
return collectGifFrames(input, null);
}
function transformGifFrames(input, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
return collectGifFrames(input, {x, y, width, height, rotationDeg, resizeWidth, resizeHeight});
}
function exactGifFrameData(rgba) {
const palette = [];
const colorToIndex = new Map();
const index = new Uint8Array(rgba.length / 4);
let transparentIndex = -1;
for (let offset = 0, pixel = 0; offset < rgba.length; offset += 4, pixel += 1) {
if (rgba[offset + 3] === 0) {
if (transparentIndex === -1) {
if (palette.length >= 256) return null;
transparentIndex = palette.length;
palette.push([0, 0, 0]);
}
index[pixel] = transparentIndex;
continue;
}
const key = `${rgba[offset]},${rgba[offset + 1]},${rgba[offset + 2]}`;
let paletteIndex = colorToIndex.get(key);
if (paletteIndex == null) {
if (palette.length >= 256) return null;
paletteIndex = palette.length;
colorToIndex.set(key, paletteIndex);
palette.push([rgba[offset], rgba[offset + 1], rgba[offset + 2]]);
}
index[pixel] = paletteIndex;
}
if (palette.length === 0) palette.push([0, 0, 0]);
return {index, palette, transparentIndex};
}
function quantizedGifFrameData(rgba) {
const input = rgba.byteOffset === 0 && rgba.byteLength === rgba.buffer.byteLength ? rgba : new Uint8Array(rgba);
const palette = quantize(input, 256, {format: 'rgba4444', oneBitAlpha: true});
const index = applyPalette(input, palette, 'rgba4444');
const transparentIndex = palette.findIndex((color) => color.length >= 4 && color[3] === 0);
const rgbPalette = palette.map((color) => [color[0], color[1], color[2]]);
return {index, palette: rgbPalette, transparentIndex};
}
function writeGifFrame(gif, frame, width, height, first) {
if (frame.width !== width || frame.height !== height) throw new Error('GIF frame dimensions must match');
const frameData = exactGifFrameData(frame.rgba) ?? quantizedGifFrameData(frame.rgba);
const transparent = frameData.transparentIndex >= 0;
gif.writeFrame(frameData.index, width, height, {
palette: frameData.palette,
delay: Math.max(0, Math.round(frame.delayMs || 0)),
repeat: 0,
transparent,
transparentIndex: transparent ? frameData.transparentIndex : 0,
first,
});
}
function encodeGifFrames(frames) {
if (!frames.length) throw new Error('GIF encode requires at least one frame');
const width = frames[0].width;
const height = frames[0].height;
const gif = GIFEncoder();
for (const frame of frames) writeGifFrame(gif, frame, width, height, false);
gif.finish();
return gif.bytes();
}
function encodeGifFrameChunk(frame, first) {
const gif = GIFEncoder({auto: false});
if (first) gif.writeHeader();
writeGifFrame(gif, frame, frame.width, frame.height, first);
return {data: gif.bytes(), width: frame.width, height: frame.height};
}
function assembleGifFrameChunks(chunks) {
if (!chunks.length) throw new Error('GIF chunk assembly requires at least one frame');
const width = chunks[0].width;
const height = chunks[0].height;
const parts = [];
for (const chunk of chunks) {
if (chunk.width !== width || chunk.height !== height) throw new Error('GIF frame dimensions must match');
parts.push(inputBytes(chunk.data));
}
parts.push(new Uint8Array([0x3b]));
return concatBytes(parts);
}
function writeU32BE(bytes, offset, value) {
bytes[offset] = (value >>> 24) & 0xff;
bytes[offset + 1] = (value >>> 16) & 0xff;
bytes[offset + 2] = (value >>> 8) & 0xff;
bytes[offset + 3] = value & 0xff;
}
function writeU16BE(bytes, offset, value) {
bytes[offset] = (value >>> 8) & 0xff;
bytes[offset + 1] = value & 0xff;
}
let pngCrcTable = null;
function crc32(bytes, start, end) {
if (!pngCrcTable) {
pngCrcTable = new Uint32Array(256);
for (let n = 0; n < 256; n += 1) {
let c = n;
for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
pngCrcTable[n] = c >>> 0;
}
}
let c = 0xffffffff;
for (let index = start; index < end; index += 1) c = pngCrcTable[(c ^ bytes[index]) & 0xff] ^ (c >>> 8);
return (c ^ 0xffffffff) >>> 0;
}
function pngChunk(type, payload) {
const typeBytes = textEncoder.encode(type);
const chunk = new Uint8Array(12 + payload.length);
writeU32BE(chunk, 0, payload.length);
chunk.set(typeBytes, 4);
chunk.set(payload, 8);
writeU32BE(chunk, 8 + payload.length, crc32(chunk, 4, 8 + payload.length));
return chunk;
}
function concatBytes(parts) {
let total = 0;
for (const part of parts) total += part.length;
const output = new Uint8Array(total);
let offset = 0;
for (const part of parts) {
output.set(part, offset);
offset += part.length;
}
return output;
}
function pngScanlines(rgba, width, height) {
const rowBytes = width * 4;
const output = new Uint8Array((rowBytes + 1) * height);
for (let row = 0; row < height; row += 1) {
const target = row * (rowBytes + 1);
output[target] = 0;
output.set(rgba.subarray(row * rowBytes, row * rowBytes + rowBytes), target + 1);
}
return output;
}
function delayFraction(delayMs) {
const ms = Math.max(0, Math.round(Number(delayMs) || 0));
if (ms === 0) return [0, 100];
if (ms <= 655350) return [Math.min(65535, Math.max(1, Math.round(ms / 10))), 100];
return [Math.min(65535, Math.max(1, Math.round(ms / 1000))), 1];
}
function encodeApngFramePayload(frame) {
return {
compressed: zlibSync(pngScanlines(frame.rgba, frame.width, frame.height), {level: 6}),
width: frame.width,
height: frame.height,
delayMs: frame.delayMs,
};
}
function assembleApngFramePayloads(frames) {
if (!frames.length) throw new Error('APNG encode requires at least one frame');
const width = frames[0].width;
const height = frames[0].height;
const chunks = [new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])];
const ihdr = new Uint8Array(13);
writeU32BE(ihdr, 0, width);
writeU32BE(ihdr, 4, height);
ihdr[8] = 8;
ihdr[9] = 6;
chunks.push(pngChunk('IHDR', ihdr));
const actl = new Uint8Array(8);
writeU32BE(actl, 0, frames.length);
writeU32BE(actl, 4, 0);
chunks.push(pngChunk('acTL', actl));
let sequence = 0;
for (let index = 0; index < frames.length; index += 1) {
const frame = frames[index];
if (frame.width !== width || frame.height !== height) throw new Error('APNG frame dimensions must match');
const fctl = new Uint8Array(26);
writeU32BE(fctl, 0, sequence);
sequence += 1;
writeU32BE(fctl, 4, width);
writeU32BE(fctl, 8, height);
writeU32BE(fctl, 12, 0);
writeU32BE(fctl, 16, 0);
const delay = delayFraction(frame.delayMs);
writeU16BE(fctl, 20, delay[0]);
writeU16BE(fctl, 22, delay[1]);
fctl[24] = 0;
fctl[25] = 0;
chunks.push(pngChunk('fcTL', fctl));
if (index === 0) {
chunks.push(pngChunk('IDAT', inputBytes(frame.compressed)));
} else {
const compressed = inputBytes(frame.compressed);
const payload = new Uint8Array(4 + compressed.length);
writeU32BE(payload, 0, sequence);
sequence += 1;
payload.set(compressed, 4);
chunks.push(pngChunk('fdAT', payload));
}
}
chunks.push(pngChunk('IEND', new Uint8Array()));
return concatBytes(chunks);
}
function encodeApngFrames(frames) {
if (!frames.length) throw new Error('APNG encode requires at least one frame');
const width = frames[0].width;
const height = frames[0].height;
for (const frame of frames) {
if (frame.width !== width || frame.height !== height) throw new Error('APNG frame dimensions must match');
}
return assembleApngFramePayloads(frames.map(encodeApngFramePayload));
}
function encodeStaticFrame(frame, outputFormat) {
switch (outputFormat) {
case 'png':
return encodePng({width: frame.width, height: frame.height, data: frame.rgba, depth: 8, channels: 4});
case 'jpeg':
return encodeJpeg({width: frame.width, height: frame.height, data: frame.rgba}, 92).data;
case 'gif':
return encodeGifFrames([frame]);
default:
throw new Error(`Unsupported static output format: ${outputFormat}`);
}
}
function decodeStaticImage(input, inputFormat) {
switch (inputFormat) {
case 'png':
return decodePngFrames(input)[0];
case 'jpeg':
return decodeJpegFrame(input);
case 'gif':
return decodeGifFrames(input)[0];
default:
throw new Error(`Unsupported static input format: ${inputFormat}`);
}
}
function normalizeFrameInput(frame) {
if (!frame || frame.rgba == null) throw new Error('Frame is missing RGBA data');
return {
rgba: inputBytes(frame.rgba),
width: nonNegativeU32(frame.width),
height: nonNegativeU32(frame.height),
delayMs: Math.max(0, Math.round(Number(frame.delayMs) || 0)),
};
}
function normalizeApngPayloadInput(frame) {
if (!frame || frame.compressed == null) throw new Error('APNG frame is missing compressed data');
return {
compressed: inputBytes(frame.compressed),
width: nonNegativeU32(frame.width),
height: nonNegativeU32(frame.height),
delayMs: Math.max(0, Math.round(Number(frame.delayMs) || 0)),
};
}
function normalizeGifChunkInput(chunk) {
if (!chunk || chunk.data == null) throw new Error('GIF frame chunk is missing data');
return {
data: inputBytes(chunk.data),
width: nonNegativeU32(chunk.width),
height: nonNegativeU32(chunk.height),
};
}
export function decode_gif_frames(input) {
return decodeGifFrames(input);
}
export function decode_apng_frames(input) {
return decodePngFrames(input);
}
export function encode_gif_frames(frames) {
return encodeGifFrames(frames.map(normalizeFrameInput));
}
export function encode_apng_frames(frames) {
return encodeApngFrames(frames.map(normalizeFrameInput));
}
export function encode_gif_frame_chunk(frame, first) {
return encodeGifFrameChunk(normalizeFrameInput(frame), Boolean(first));
}
export function assemble_gif_frame_chunks(chunks) {
return assembleGifFrameChunks(chunks.map(normalizeGifChunkInput));
}
export function encode_apng_frame_payload(frame) {
return encodeApngFramePayload(normalizeFrameInput(frame));
}
export function assemble_apng_frames(frames) {
return assembleApngFramePayloads(frames.map(normalizeApngPayloadInput));
}
export function crop_and_rotate_apng(input, x, y, width, height, rotation_deg, resize_width, resize_height) {
const bytes = inputBytes(input);
const dimensions = pngDimensions(bytes);
if (
dimensions &&
isNoopTransform(dimensions.width, dimensions.height, x, y, width, height, rotation_deg, resize_width, resize_height)
) {
return bytes.slice();
}
const frames = decodePngFrames(bytes);
if (
isNoopTransform(frames[0].width, frames[0].height, x, y, width, height, rotation_deg, resize_width, resize_height)
) {
return bytes.slice();
}
return encodeApngFrames(transformFrames(frames, x, y, width, height, rotation_deg, resize_width, resize_height));
}
export function crop_and_rotate_gif(input, x, y, width, height, rotation_deg, resize_width, resize_height) {
const bytes = inputBytes(input);
const dimensions = gifDimensions(bytes);
if (
dimensions &&
isNoopTransform(dimensions.width, dimensions.height, x, y, width, height, rotation_deg, resize_width, resize_height)
) {
return bytes.slice();
}
return encodeGifFrames(transformGifFrames(bytes, x, y, width, height, rotation_deg, resize_width, resize_height));
}
export function crop_and_rotate_image(
input,
format_hint,
x,
y,
width,
height,
rotation_deg,
resize_width,
resize_height,
) {
const bytes = inputBytes(input);
const inputFormat = sniffImageFormat(bytes);
const requestedFormat = normalizedFormat(format_hint);
const outputFormat = requestedFormat && requestedFormat !== 'unknown' ? requestedFormat : inputFormat;
if (inputFormat === 'webp' || inputFormat === 'avif' || outputFormat === 'webp' || outputFormat === 'avif') {
throw new Error('WebP and AVIF crop/encode use the browser or native media bridge');
}
if (inputFormat === outputFormat) {
const dimensions = imageDimensions(bytes, inputFormat);
if (
dimensions &&
isNoopTransform(
dimensions.width,
dimensions.height,
x,
y,
width,
height,
rotation_deg,
resize_width,
resize_height,
)
) {
return bytes.slice();
}
}
const frame = decodeStaticImage(bytes, inputFormat);
if (
inputFormat === outputFormat &&
isNoopTransform(frame.width, frame.height, x, y, width, height, rotation_deg, resize_width, resize_height)
) {
return bytes.slice();
}
return encodeStaticFrame(
transformFrame(frame, x, y, width, height, rotation_deg, resize_width, resize_height),
outputFormat,
);
}
@@ -0,0 +1,13 @@
[package]
name = "fluxer-content-update-frozen-snapshot"
version = "0.1.0"
edition.workspace = true
license.workspace = true
[dependencies]
anyhow = "1.0.102"
base64 = "0.22.1"
sha2 = "0.11.0"
[dev-dependencies]
tempfile = "3.27.0"
@@ -0,0 +1,158 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::{Context, Result, bail};
use base64::Engine;
use sha2::{Digest, Sha256};
use std::fs;
use std::path::Path;
pub fn generate_snapshot_source(static_dir: &Path) -> Result<String> {
if !static_dir.is_dir() {
bail!("{} is not a directory", static_dir.display());
}
let index_html = read_file(&static_dir.join("index.html"))?;
let sw_js = read_file(&static_dir.join("sw.js"))?;
let version_json = read_file(&static_dir.join("version.json"))?;
let sha = snapshot_sha(&index_html, &sw_js, &version_json);
let mut output = String::new();
output.push_str(&format!("const SNAPSHOT_SHA: &str = \"{sha}\";\n\n"));
output.push_str(&format!(
"// -- base64-encoded index.html ({} bytes) --\n",
index_html.len()
));
output.push_str(&format_const(
"STABLE_INDEX_HTML",
&base64_standard(&index_html),
));
output.push_str(&format!(
"// -- base64-encoded sw.js ({} bytes) --\n",
sw_js.len()
));
output.push_str(&format_const("STABLE_SW_JS", &base64_standard(&sw_js)));
output.push_str(&format!(
"// -- base64-encoded version.json ({} bytes) --\n",
version_json.len()
));
output.push_str(&format_const(
"STABLE_VERSION_JSON",
&base64_standard(&version_json),
));
Ok(output)
}
fn read_file(path: &Path) -> Result<Vec<u8>> {
if !path.exists() {
bail!("{} does not exist", path.display());
}
fs::read(path).with_context(|| format!("failed to read {}", path.display()))
}
fn snapshot_sha(index_html: &[u8], sw_js: &[u8], version_json: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(index_html);
hasher.update(sw_js);
hasher.update(version_json);
bytes_to_lower_hex(&hasher.finalize())
}
fn bytes_to_lower_hex(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for byte in bytes {
out.push(HEX[(byte >> 4) as usize] as char);
out.push(HEX[(byte & 0x0f) as usize] as char);
}
out
}
fn base64_standard(data: &[u8]) -> String {
base64::engine::general_purpose::STANDARD.encode(data)
}
fn break_lines(value: &str, width: usize) -> Vec<&str> {
value
.as_bytes()
.chunks(width)
.map(|chunk| std::str::from_utf8(chunk).expect("base64 should be ascii"))
.collect()
}
fn format_const(name: &str, data: &str) -> String {
let escaped = break_lines(data, 100).join("\\\n");
format!("const {name}: &str = \"\\\n{escaped}\\\n\";\n")
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::tempdir;
#[test]
fn emits_snapshot_constants_with_combined_sha() {
let dir = tempdir().unwrap();
let index_html = b"<!doctype html><html><body>fixture</body></html>";
let sw_js = b"\"use strict\";\nself.addEventListener('install', () => {});\n";
let version_json = br#"{"sha":"fixture","buildNumber":7}"#;
fs::write(dir.path().join("index.html"), index_html).unwrap();
fs::write(dir.path().join("sw.js"), sw_js).unwrap();
fs::write(dir.path().join("version.json"), version_json).unwrap();
let output = generate_snapshot_source(dir.path()).unwrap();
let expected_sha = snapshot_sha(index_html, sw_js, version_json);
assert!(output.starts_with(&format!(
"const SNAPSHOT_SHA: &str = \"{expected_sha}\";\n\n"
)));
assert!(output.contains("// -- base64-encoded index.html (48 bytes) --"));
assert!(output.contains("// -- base64-encoded sw.js (58 bytes) --"));
assert!(output.contains("// -- base64-encoded version.json (33 bytes) --"));
assert!(output.contains(&format_const(
"STABLE_INDEX_HTML",
&base64_standard(index_html)
)));
assert!(output.contains(&format_const("STABLE_SW_JS", &base64_standard(sw_js))));
assert!(output.contains(&format_const(
"STABLE_VERSION_JSON",
&base64_standard(version_json)
)));
}
#[test]
fn wraps_base64_constants_at_100_columns() {
let data = "a".repeat(205);
let formatted = format_const("TEST", &data);
let lines = formatted.lines().collect::<Vec<_>>();
assert_eq!(lines[0], "const TEST: &str = \"\\");
assert_eq!(lines[1], format!("{}\\", "a".repeat(100)));
assert_eq!(lines[2], format!("{}\\", "a".repeat(100)));
assert_eq!(lines[3], format!("{}\\", "a".repeat(5)));
assert_eq!(lines[4], "\";");
}
#[test]
fn reports_non_directory_static_path() {
let dir = tempdir().unwrap();
let err = generate_snapshot_source(&dir.path().join("missing"))
.unwrap_err()
.to_string();
assert!(err.contains("missing is not a directory"));
}
#[test]
fn reports_missing_required_file() {
let dir = tempdir().unwrap();
fs::write(dir.path().join("index.html"), b"index").unwrap();
let err = generate_snapshot_source(dir.path())
.unwrap_err()
.to_string();
assert!(err.contains("sw.js does not exist"));
}
}
@@ -0,0 +1,36 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::Result;
use fluxer_content_update_frozen_snapshot::generate_snapshot_source;
use std::env;
use std::path::PathBuf;
use std::process;
fn main() {
if let Err(err) = run() {
eprintln!("error: {err:#}");
process::exit(1);
}
}
fn run() -> Result<()> {
let mut args = env::args_os();
let program = args
.next()
.and_then(|value| PathBuf::from(value).file_name().map(|name| name.to_owned()))
.and_then(|name| name.into_string().ok())
.unwrap_or_else(|| "fluxer-content-update-frozen-snapshot".to_owned());
let Some(static_dir) = args.next() else {
eprintln!("usage: {program} <static_dir>");
process::exit(1);
};
if args.next().is_some() {
eprintln!("usage: {program} <static_dir>");
process::exit(1);
}
let output = generate_snapshot_source(&PathBuf::from(static_dir))?;
print!("{output}");
Ok(())
}
+31
View File
@@ -0,0 +1,31 @@
[package]
name = "fluxer-dev"
version = "0.1.0"
edition.workspace = true
license.workspace = true
[dependencies]
anyhow = "1.0.102"
axum = "0.8.9"
base64 = "0.22.1"
chrono = "0.4.45"
clap = { version = "4.6.1", features = ["derive", "env"] }
futures-util = "0.3.32"
hmac = "0.13.0"
hyper = { version = "1.10.1", features = ["http1", "server"] }
hyper-util = { version = "0.1.20", features = ["tokio"] }
image = { version = "0.25.10", default-features = false, features = ["gif", "jpeg", "png", "webp"] }
libc = "0.2.186"
regex = "1.12"
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] }
scylla = { version = "1.6.0", features = ["chrono-04"] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.150"
sha2 = "0.11.0"
tempfile = "3.27.0"
tokio = { version = "1.52.3", features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "time"] }
tokio-tungstenite = { version = "0.29.0", features = ["rustls-tls-native-roots"] }
url = "2.5.8"
urlencoding = "2.1.3"
[dev-dependencies]
File diff suppressed because it is too large Load Diff
+58
View File
@@ -0,0 +1,58 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::desktop::install_desktop;
use crate::gateway::setup_gateway_config;
use crate::paths::{ensure_state_dirs, ensure_writable_dev_paths};
use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command, wait_http, wait_tcp};
use crate::smoke::{bootstrap_schema_and_object_store, run_smoke, wait_s3_api};
use anyhow::Result;
pub async fn bootstrap(skip_install: bool, skip_desktop_install: bool) -> Result<()> {
ensure_state_dirs()?;
ensure_writable_dev_paths()?;
if !skip_install {
crate::proc::run(&["corepack", "enable"])?;
crate::proc::run(&["corepack", "prepare", "pnpm@10.29.3", "--activate"])?;
run_command(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
env: PNPM_INSTALL_ENV
.iter()
.map(|(key, value)| ((*key).to_owned(), Some((*value).to_owned())))
.collect(),
..RunOptions::default()
},
)?;
if !skip_desktop_install {
install_desktop()?;
}
}
setup_gateway_config()?;
wait_core_infra().await?;
bootstrap_schema_and_object_store().await?;
run_smoke(false, false).await?;
println!("Fluxer dev bootstrap complete.");
Ok(())
}
pub async fn post_start() -> Result<()> {
ensure_state_dirs()?;
ensure_writable_dev_paths()?;
setup_gateway_config()?;
run_smoke(true, false).await
}
pub async fn wait_core_infra() -> Result<()> {
wait_tcp("Valkey", "valkey", 6379, 120).await?;
wait_tcp("NATS", "nats", 4222, 120).await?;
wait_tcp("LiveKit", "livekit", 7880, 120).await?;
crate::media_proxy::ensure_dev_object_store(true, 120).await?;
wait_tcp("SeaweedFS S3", "127.0.0.1", 8333, 120).await?;
wait_http(
"SeaweedFS master",
"http://127.0.0.1:9333/cluster/status",
120,
)
.await?;
wait_s3_api(120).await
}
+706
View File
@@ -0,0 +1,706 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::paths::DEV_CASSANDRA_DIR;
use anyhow::{Context, Result, bail};
use regex::Regex;
use scylla::DeserializeRow;
use scylla::client::session::Session;
use scylla::client::session_builder::SessionBuilder;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use std::time::{Duration, Instant};
use tokio::time::sleep;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CassandraConfig {
pub host: String,
pub port: u16,
pub keyspace: String,
pub local_dc: String,
pub username: Option<String>,
pub password: Option<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ExistingTable {
pub columns: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ExistingSchema {
pub keyspace_exists: bool,
pub types: BTreeMap<String, BTreeMap<String, String>>,
pub tables: BTreeMap<String, ExistingTable>,
pub indexes: BTreeSet<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SchemaDiff {
pub statements: Vec<String>,
pub warnings: Vec<String>,
pub errors: Vec<String>,
pub target_sha256: String,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct Field {
pub name: String,
#[serde(rename = "type")]
pub field_type: String,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct UserType {
pub name: String,
pub fields: Vec<Field>,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct Table {
pub name: String,
pub columns: Vec<Field>,
pub primary_key: String,
pub options: String,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct Index {
pub name: String,
pub table: String,
pub expression: String,
}
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub struct TargetSchema {
pub user_types: Vec<UserType>,
pub tables: Vec<Table>,
pub indexes: Vec<Index>,
}
static TARGET_SCHEMA: LazyLock<TargetSchema> = LazyLock::new(|| {
serde_json::from_str(include_str!("../cassandra_target_schema.json"))
.expect("embedded Cassandra target schema JSON is valid")
});
static TYPE_TOKEN_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[a-zA-Z_][a-zA-Z0-9_]*").expect("valid type token regex"));
static WHITESPACE_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\s+").expect("valid whitespace regex"));
pub fn target_schema() -> &'static TargetSchema {
&TARGET_SCHEMA
}
pub fn config_from_env() -> Result<CassandraConfig> {
let hosts = env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned());
let host = hosts
.split_once(',')
.map(|(first, _)| first)
.unwrap_or(&hosts)
.trim();
Ok(CassandraConfig {
host: if host.is_empty() {
"cassandra".to_owned()
} else {
host.to_owned()
},
port: env::var("FLUXER_CASSANDRA_PORT")
.unwrap_or_else(|_| "9042".to_owned())
.parse()
.context("invalid FLUXER_CASSANDRA_PORT")?,
keyspace: env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned()),
local_dc: env::var("FLUXER_CASSANDRA_LOCAL_DC")
.unwrap_or_else(|_| "datacenter1".to_owned()),
username: env::var("FLUXER_CASSANDRA_USERNAME")
.ok()
.filter(|value| !value.is_empty()),
password: env::var("FLUXER_CASSANDRA_PASSWORD")
.ok()
.filter(|value| !value.is_empty()),
})
}
async fn connect(config: &CassandraConfig) -> Result<Session> {
let contact = format!("{}:{}", config.host, config.port);
let mut builder = SessionBuilder::new().known_node(contact);
if let Some(username) = &config.username {
builder = builder.user(username, config.password.as_deref().unwrap_or_default());
}
Ok(builder.build().await?)
}
pub async fn wait_for_cassandra(config: &CassandraConfig, timeout_secs: u64) -> Result<()> {
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_error = None;
while Instant::now() < deadline {
match connect(config).await {
Ok(session) => match session
.query_unpaged("SELECT release_version FROM system.local", ())
.await
{
Ok(_) => {
println!("Cassandra is reachable at {}:{}", config.host, config.port);
return Ok(());
}
Err(error) => last_error = Some(error.to_string()),
},
Err(error) => last_error = Some(error.to_string()),
}
sleep(Duration::from_secs(3)).await;
}
bail!(
"Timed out waiting for Cassandra at {}:{}: {}",
config.host,
config.port,
last_error.unwrap_or_else(|| "unknown error".to_owned())
);
}
pub fn normalize_type(value: &str) -> String {
let mut normalized = WHITESPACE_RE.replace_all(value, "").to_ascii_lowercase();
let user_type_names = target_schema()
.user_types
.iter()
.map(|user_type| user_type.name.as_str())
.collect::<Vec<_>>();
let mut previous = String::new();
while previous != normalized {
previous = normalized.clone();
for name in &user_type_names {
normalized = normalized.replace(&format!("frozen<{name}>"), name);
}
}
normalized
}
fn normalize_identifier(value: &str) -> String {
value.trim_matches('"').to_ascii_lowercase()
}
#[derive(Debug, DeserializeRow)]
#[allow(dead_code)]
struct KeyspaceRow {
keyspace_name: String,
}
#[derive(Debug, DeserializeRow)]
struct TypeRow {
type_name: String,
field_names: Option<Vec<String>>,
field_types: Option<Vec<String>>,
}
#[derive(Debug, DeserializeRow)]
struct ColumnRow {
table_name: String,
column_name: String,
#[scylla(rename = "type")]
column_type: String,
}
#[derive(Debug, DeserializeRow)]
struct IndexRow {
index_name: Option<String>,
}
pub async fn fetch_existing_schema(session: &Session, keyspace: &str) -> Result<ExistingSchema> {
let keyspace_rows = session
.query_unpaged(
"SELECT keyspace_name FROM system_schema.keyspaces WHERE keyspace_name = ?",
(keyspace,),
)
.await?
.into_rows_result()?
.rows::<KeyspaceRow>()?
.collect::<std::result::Result<Vec<_>, _>>()?;
let type_rows = session
.query_unpaged(
"SELECT type_name, field_names, field_types FROM system_schema.types WHERE keyspace_name = ?",
(keyspace,),
)
.await?
.into_rows_result()?
.rows::<TypeRow>()?
.collect::<std::result::Result<Vec<_>, _>>()?;
let mut types = BTreeMap::new();
for row in type_rows {
let mut fields = BTreeMap::new();
for (name, field_type) in row
.field_names
.unwrap_or_default()
.into_iter()
.zip(row.field_types.unwrap_or_default())
{
fields.insert(normalize_identifier(&name), field_type);
}
types.insert(normalize_identifier(&row.type_name), fields);
}
let column_rows = session
.query_unpaged(
"SELECT table_name, column_name, type FROM system_schema.columns WHERE keyspace_name = ?",
(keyspace,),
)
.await?
.into_rows_result()?
.rows::<ColumnRow>()?
.collect::<std::result::Result<Vec<_>, _>>()?;
let mut tables: BTreeMap<String, ExistingTable> = BTreeMap::new();
for row in column_rows {
tables
.entry(normalize_identifier(&row.table_name))
.or_default()
.columns
.insert(normalize_identifier(&row.column_name), row.column_type);
}
let index_rows = session
.query_unpaged(
"SELECT index_name FROM system_schema.indexes WHERE keyspace_name = ?",
(keyspace,),
)
.await?
.into_rows_result()?
.rows::<IndexRow>()?
.collect::<std::result::Result<Vec<_>, _>>()?;
let indexes = index_rows
.into_iter()
.filter_map(|row| row.index_name)
.map(|name| normalize_identifier(&name))
.collect();
Ok(ExistingSchema {
keyspace_exists: !keyspace_rows.is_empty(),
types,
tables,
indexes,
})
}
pub fn render_user_type(user_type: &UserType, keyspace: &str) -> String {
let fields = user_type
.fields
.iter()
.map(|field| format!("{} {}", field.name, field.field_type))
.collect::<Vec<_>>()
.join(",\n ");
format!(
"CREATE TYPE IF NOT EXISTS {keyspace}.{} (\n {fields}\n)",
user_type.name
)
}
pub fn render_table(table: &Table, keyspace: &str) -> String {
let mut fields = table
.columns
.iter()
.map(|field| format!("{} {}", field.name, field.field_type))
.collect::<Vec<_>>();
fields.push(format!("PRIMARY KEY {}", table.primary_key));
let body = fields.join(",\n ");
let mut statement = format!(
"CREATE TABLE IF NOT EXISTS {keyspace}.{} (\n {body}\n)",
table.name
);
if !table.options.is_empty() {
statement.push_str(" WITH ");
statement.push_str(&table.options);
}
statement
}
pub fn render_index(index: &Index, keyspace: &str) -> String {
format!(
"CREATE INDEX IF NOT EXISTS {} ON {keyspace}.{} ({})",
index.name, index.table, index.expression
)
}
pub fn render_target_schema(keyspace: &str) -> String {
let schema = target_schema();
let mut statements = Vec::new();
for user_type in sorted_user_types() {
statements.push(statement_with_semicolon(&render_user_type(
&user_type, keyspace,
)));
}
let mut tables = schema.tables.clone();
tables.sort_by(|a, b| a.name.cmp(&b.name));
for table in tables {
statements.push(statement_with_semicolon(&render_table(&table, keyspace)));
}
let mut indexes = schema.indexes.clone();
indexes.sort_by(|a, b| a.name.cmp(&b.name));
for index in indexes {
statements.push(statement_with_semicolon(&render_index(&index, keyspace)));
}
format!("{}\n", statements.join("\n\n"))
}
pub fn target_schema_sha256(keyspace: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(render_target_schema(keyspace).as_bytes());
bytes_to_lower_hex(&hasher.finalize())
}
fn bytes_to_lower_hex(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
for byte in bytes {
out.push(HEX[(byte >> 4) as usize] as char);
out.push(HEX[(byte & 0x0f) as usize] as char);
}
out
}
pub fn sorted_user_types() -> Vec<UserType> {
let schema = target_schema();
let by_name = schema
.user_types
.iter()
.map(|user_type| (user_type.name.clone(), user_type.clone()))
.collect::<BTreeMap<_, _>>();
let mut done = BTreeSet::new();
let mut result = Vec::new();
let mut remaining = by_name.clone();
while !remaining.is_empty() {
let mut progressed = false;
for (name, user_type) in remaining.clone() {
let deps = type_dependencies(&user_type, &by_name.keys().cloned().collect());
if deps.is_subset(&done) {
result.push(user_type);
done.insert(name.clone());
remaining.remove(&name);
progressed = true;
}
}
if !progressed {
result.extend(remaining.into_values());
break;
}
}
result
}
fn type_dependencies(user_type: &UserType, user_type_names: &BTreeSet<String>) -> BTreeSet<String> {
let mut deps = BTreeSet::new();
for field in &user_type.fields {
for token in TYPE_TOKEN_RE.find_iter(&field.field_type) {
let token = token.as_str().to_ascii_lowercase();
if user_type_names.contains(&token) && token != user_type.name {
deps.insert(token);
}
}
}
deps
}
pub fn create_diff(existing: &ExistingSchema, keyspace: &str) -> SchemaDiff {
let schema = target_schema();
let mut statements = Vec::new();
let mut warnings = Vec::new();
let mut errors = Vec::new();
if !existing.keyspace_exists {
statements.push(format!(
"CREATE KEYSPACE IF NOT EXISTS {keyspace} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}"
));
}
for user_type in sorted_user_types() {
let Some(current_fields) = existing.types.get(&user_type.name) else {
statements.push(render_user_type(&user_type, keyspace));
continue;
};
for field in &user_type.fields {
match current_fields.get(&field.name) {
None => statements.push(format!(
"ALTER TYPE {keyspace}.{} ADD IF NOT EXISTS {} {}",
user_type.name, field.name, field.field_type
)),
Some(current_type)
if normalize_type(current_type) != normalize_type(&field.field_type) =>
{
errors.push(format!(
"type {}.{} has {current_type:?}, expected {:?}",
user_type.name, field.name, field.field_type
));
}
Some(_) => {}
}
}
}
let mut tables = schema.tables.clone();
tables.sort_by(|a, b| a.name.cmp(&b.name));
for table in tables {
let Some(current_table) = existing.tables.get(&table.name) else {
statements.push(render_table(&table, keyspace));
continue;
};
for field in &table.columns {
match current_table.columns.get(&field.name) {
None => statements.push(format!(
"ALTER TABLE {keyspace}.{} ADD IF NOT EXISTS {} {}",
table.name, field.name, field.field_type
)),
Some(current_type)
if normalize_type(current_type) != normalize_type(&field.field_type) =>
{
errors.push(format!(
"table {}.{} has {current_type:?}, expected {:?}",
table.name, field.name, field.field_type
));
}
Some(_) => {}
}
}
}
let mut indexes = schema.indexes.clone();
indexes.sort_by(|a, b| a.name.cmp(&b.name));
for index in indexes {
if !existing.indexes.contains(&index.name.to_ascii_lowercase()) {
statements.push(render_index(&index, keyspace));
}
}
let target_tables = schema
.tables
.iter()
.map(|table| table.name.clone())
.collect::<BTreeSet<_>>();
for table_name in existing.tables.keys() {
if !target_tables.contains(table_name) && !table_name.starts_with("system_") {
warnings.push(format!(
"extra Cassandra table exists outside target schema: {table_name}"
));
}
}
SchemaDiff {
statements,
warnings,
errors,
target_sha256: target_schema_sha256(keyspace),
}
}
fn statement_with_semicolon(statement: &str) -> String {
format!("{};", statement.trim().trim_end_matches(';'))
}
pub fn render_diff(diff: &SchemaDiff) -> String {
let mut lines = vec![
"-- Generated by fluxer-dev cassandra diff".to_owned(),
format!("-- Target schema sha256: {}", diff.target_sha256),
String::new(),
];
if !diff.errors.is_empty() {
lines.push("-- Diff contains validation errors and should not be applied:".to_owned());
lines.extend(diff.errors.iter().map(|error| format!("-- {error}")));
lines.push(String::new());
}
if !diff.warnings.is_empty() {
lines.push("-- Warnings:".to_owned());
lines.extend(
diff.warnings
.iter()
.map(|warning| format!("-- {warning}")),
);
lines.push(String::new());
}
if diff.statements.is_empty() {
lines.push("-- No schema changes required.".to_owned());
} else {
lines.extend(
diff.statements
.iter()
.map(|statement| statement_with_semicolon(statement)),
);
}
format!("{}\n", lines.join("\n"))
}
pub fn write_diff_file(diff: &SchemaDiff, path: Option<&Path>) -> Result<PathBuf> {
std::fs::create_dir_all(DEV_CASSANDRA_DIR.as_path())?;
let output = path
.map(Path::to_path_buf)
.unwrap_or_else(|| DEV_CASSANDRA_DIR.join("last-diff.cql"));
std::fs::write(&output, render_diff(diff))?;
std::fs::write(
DEV_CASSANDRA_DIR.join("target-schema.sha256"),
format!("{}\n", diff.target_sha256),
)?;
Ok(output)
}
pub async fn compute_diff(config: Option<CassandraConfig>) -> Result<SchemaDiff> {
let config = config.unwrap_or(config_from_env()?);
wait_for_cassandra(&config, 180).await?;
let session = connect(&config).await?;
create_live_diff(&session, &config.keyspace).await
}
async fn create_live_diff(session: &Session, keyspace: &str) -> Result<SchemaDiff> {
Ok(create_diff(
&fetch_existing_schema(session, keyspace).await?,
keyspace,
))
}
pub async fn apply_schema(config: Option<CassandraConfig>) -> Result<SchemaDiff> {
let config = config.unwrap_or(config_from_env()?);
wait_for_cassandra(&config, 180).await?;
let session = connect(&config).await?;
let diff = create_live_diff(&session, &config.keyspace).await?;
let diff_path = write_diff_file(&diff, None)?;
if !diff.errors.is_empty() {
bail!(
"Cassandra schema has validation errors; wrote {}",
diff_path.display()
);
}
for warning in &diff.warnings {
println!("Cassandra schema warning: {warning}");
}
if diff.statements.is_empty() {
println!(
"Cassandra schema is already up to date; wrote {}",
diff_path.display()
);
} else {
println!(
"Applying {} Cassandra schema statement(s); wrote {}",
diff.statements.len(),
diff_path.display()
);
for statement in &diff.statements {
session.query_unpaged(statement.as_str(), ()).await?;
}
}
verify_schema_with_session(&session, &config.keyspace).await?;
Ok(diff)
}
pub async fn verify_schema(
config: Option<CassandraConfig>,
keyspace: Option<String>,
) -> Result<()> {
let config = config.unwrap_or(config_from_env()?);
let keyspace = keyspace.unwrap_or_else(|| config.keyspace.clone());
wait_for_cassandra(&config, 180).await?;
let session = connect(&config).await?;
verify_schema_with_session(&session, &keyspace).await
}
async fn verify_schema_with_session(session: &Session, keyspace: &str) -> Result<()> {
let existing = fetch_existing_schema(session, keyspace).await?;
let diff = create_diff(&existing, keyspace);
let unapplied = diff.statements.clone();
if !diff.errors.is_empty() || !unapplied.is_empty() {
write_diff_file(&diff, None)?;
let mut problems = diff.errors;
problems.extend(
unapplied
.into_iter()
.take(10)
.map(|statement| format!("still needs: {statement}")),
);
bail!(
"Cassandra schema verification failed:\n{}",
problems
.into_iter()
.map(|item| format!(" - {item}"))
.collect::<Vec<_>>()
.join("\n")
);
}
println!(
"Cassandra schema verified: {} types, {} tables, {} indexes, target {}",
target_schema().user_types.len(),
target_schema().tables.len(),
target_schema().indexes.len(),
&diff.target_sha256[..12],
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalizes_frozen_user_type_wrappers() {
assert_eq!(normalize_type(" frozen<message_embed> "), "message_embed");
assert_eq!(
normalize_type("LIST< frozen<message_embed> >"),
"list<message_embed>"
);
}
#[test]
fn renders_target_schema_with_semicolons() {
let rendered = render_target_schema("fluxer");
assert!(rendered.starts_with("CREATE TYPE IF NOT EXISTS fluxer."));
assert!(rendered.contains("CREATE TABLE IF NOT EXISTS fluxer.messages"));
assert!(rendered.ends_with(";\n"));
}
#[test]
fn diff_creates_keyspace_and_missing_schema() {
let diff = create_diff(&ExistingSchema::default(), "fluxer");
assert!(diff.statements[0].starts_with("CREATE KEYSPACE IF NOT EXISTS fluxer"));
assert!(
diff.statements
.iter()
.any(|statement| statement.starts_with("CREATE TYPE IF NOT EXISTS fluxer."))
);
assert!(
diff.statements
.iter()
.any(|statement| statement.starts_with("CREATE TABLE IF NOT EXISTS fluxer."))
);
assert!(diff.errors.is_empty());
}
#[test]
fn diff_reports_type_mismatches_and_extra_tables() {
let mut existing = ExistingSchema {
keyspace_exists: true,
..ExistingSchema::default()
};
existing.types.insert(
"message_attachment".to_owned(),
BTreeMap::from([("attachment_id".to_owned(), "text".to_owned())]),
);
existing
.tables
.insert("unexpected".to_owned(), ExistingTable::default());
let diff = create_diff(&existing, "fluxer");
assert!(
diff.errors
.iter()
.any(|error| error.contains("type message_attachment.attachment_id"))
);
assert!(
diff.warnings
.iter()
.any(|warning| warning.contains("unexpected"))
);
}
#[test]
fn render_diff_includes_warnings_errors_and_sha() {
let diff = SchemaDiff {
statements: vec!["SELECT 1".to_owned()],
warnings: vec!["warn".to_owned()],
errors: vec!["err".to_owned()],
target_sha256: "abc".to_owned(),
};
let rendered = render_diff(&diff);
assert!(rendered.contains("-- Target schema sha256: abc"));
assert!(rendered.contains("-- warn"));
assert!(rendered.contains("-- err"));
assert!(rendered.ends_with("SELECT 1;\n"));
}
}
+601
View File
@@ -0,0 +1,601 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::manifest::LOCAL_APP_URL;
use crate::paths::{DESKTOP_DIR, ROOT};
use crate::proc::{PNPM_INSTALL_ENV, RunOptions, run_command};
use anyhow::{Context, Result, bail};
use std::env;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};
use url::Url;
const CANARY_APP_NAME: &str = "Fluxer Canary";
const CANARY_BUNDLE_ID: &str = "app.fluxer.canary";
const CANARY_RPC_PORT: u16 = 21864;
const MACOS_DEV_ELECTRON_USAGE_DESCRIPTIONS: &[(&str, &str)] = &[
(
"NSMicrophoneUsageDescription",
"Fluxer needs access to your microphone to enable voice chat features.",
),
(
"NSCameraUsageDescription",
"Fluxer needs access to your camera to enable video chat features.",
),
(
"NSAppleEventsUsageDescription",
"Fluxer needs access to Apple Events for automation features.",
),
(
"NSAudioCaptureUsageDescription",
"Fluxer captures audio from the screen or window you choose to share.",
),
(
"NSScreenCaptureUsageDescription",
"Fluxer captures the screen or window you choose to share.",
),
];
pub fn install_desktop() -> Result<()> {
run_command(
&["pnpm", "install", "--frozen-lockfile"],
RunOptions {
cwd: ROOT.as_path(),
env: PNPM_INSTALL_ENV
.iter()
.map(|(k, v)| ((*k).to_owned(), Some((*v).to_owned())))
.collect(),
..RunOptions::default()
},
)
.map(drop)
}
pub fn build_desktop(skip_native: bool) -> Result<()> {
let env = vec![
(
"BUILD_CHANNEL".to_owned(),
Some(env::var("BUILD_CHANNEL").unwrap_or_else(|_| "canary".to_owned())),
),
(
"FLUXER_SKIP_NATIVE".to_owned(),
if skip_native {
Some("true".to_owned())
} else {
None
},
),
(
"PUBLIC_BUILD_VERSION".to_owned(),
Some(env::var("PUBLIC_BUILD_VERSION").unwrap_or_else(|_| "dev".to_owned())),
),
(
"PUBLIC_RELEASE_CHANNEL".to_owned(),
Some(env::var("PUBLIC_RELEASE_CHANNEL").unwrap_or_else(|_| "canary".to_owned())),
),
];
run_command(
&["pnpm", "build"],
RunOptions {
cwd: DESKTOP_DIR.as_path(),
env,
..RunOptions::default()
},
)
.map(drop)
}
pub fn typecheck_desktop() -> Result<()> {
run_command(
&["pnpm", "typecheck"],
RunOptions {
cwd: DESKTOP_DIR.as_path(),
..RunOptions::default()
},
)
.map(drop)
}
pub fn electron_args(args: &[String]) -> Vec<String> {
let mut runtime_args = Vec::new();
if cfg!(target_os = "linux")
&& Path::new("/.dockerenv").exists()
&& env::var("FLUXER_ELECTRON_NO_SANDBOX").as_deref() != Ok("0")
{
runtime_args.push("--no-sandbox".to_owned());
}
runtime_args.extend(args.iter().cloned());
runtime_args
}
pub fn electron_command(args: &[String], headless: bool) -> Vec<String> {
let mut command = base_electron_command();
command.extend(electron_args(args));
if headless
&& cfg!(target_os = "linux")
&& env::var_os("DISPLAY").is_none()
&& crate::paths::which("xvfb-run").is_some()
{
let mut wrapped = vec!["xvfb-run".to_owned(), "-a".to_owned()];
wrapped.extend(command);
return wrapped;
}
command
}
fn base_electron_command() -> Vec<String> {
if cfg!(target_os = "macos") && !Path::new("/.dockerenv").exists() {
let electron_binary = dev_electron_binary_path();
if electron_binary.is_file()
&& let Ok(launcher) = env::current_exe()
{
return disclaimed_electron_command(&launcher, &electron_binary);
}
}
vec![
"pnpm".to_owned(),
"exec".to_owned(),
"electron".to_owned(),
".".to_owned(),
]
}
fn disclaimed_electron_command(launcher: &Path, electron_binary: &Path) -> Vec<String> {
vec![
launcher.to_string_lossy().into_owned(),
"desktop".to_owned(),
"exec-disclaimed".to_owned(),
electron_binary.to_string_lossy().into_owned(),
".".to_owned(),
]
}
pub fn smoke_build_desktop() -> Result<()> {
install_desktop()?;
build_desktop(true)?;
let command = electron_command(
&[
"--fluxer-debug-info".to_owned(),
format!("--fluxer-app-url={LOCAL_APP_URL}"),
],
true,
);
let args: Vec<_> = command.iter().map(String::as_str).collect();
run_command(
&args,
RunOptions {
cwd: DESKTOP_DIR.as_path(),
env: vec![("FLUXER_SKIP_NATIVE".to_owned(), Some("true".to_owned()))],
..RunOptions::default()
},
)
.map(drop)
}
pub fn package_desktop(args: &[String]) -> Result<()> {
build_desktop(false)?;
let mut builder_args = vec![
"pnpm".to_owned(),
"exec".to_owned(),
"electron-builder".to_owned(),
"--dir".to_owned(),
"--config".to_owned(),
"electron-builder.config.cjs".to_owned(),
];
builder_args.extend(args.iter().cloned());
let refs: Vec<_> = builder_args.iter().map(String::as_str).collect();
run_command(
&refs,
RunOptions {
cwd: DESKTOP_DIR.as_path(),
..RunOptions::default()
},
)
.map(drop)
}
pub fn run_desktop(app_url: &str, extra_args: &[String], build: bool) -> Result<()> {
install_desktop()?;
if build {
build_desktop(false)?;
}
run_desktop_process(app_url, extra_args)
}
pub async fn run_desktop_canary(
app_url: Option<&str>,
extra_args: &[String],
build: bool,
) -> Result<()> {
ensure_host_macos()?;
let app_url = resolve_desktop_canary_app_url(app_url).await?;
stop_running_canary_on_host()?;
install_desktop()?;
if build {
build_desktop(false)?;
}
println!("Starting {CANARY_APP_NAME} against {app_url}");
run_desktop_process(&app_url, extra_args)
}
fn run_desktop_process(app_url: &str, extra_args: &[String]) -> Result<()> {
patch_macos_dev_electron_info_plist()?;
let mut args = vec![
format!("--fluxer-app-url={app_url}"),
"--fluxer-log-renderer-console".to_owned(),
];
args.extend(extra_args.iter().cloned());
let command = electron_command(&args, false);
let refs: Vec<_> = command.iter().map(String::as_str).collect();
run_command(
&refs,
RunOptions {
cwd: DESKTOP_DIR.as_path(),
env: vec![
("BUILD_CHANNEL".to_owned(), Some("canary".to_owned())),
(
"PUBLIC_BUILD_VERSION".to_owned(),
Some(env::var("PUBLIC_BUILD_VERSION").unwrap_or_else(|_| "dev".to_owned())),
),
(
"PUBLIC_RELEASE_CHANNEL".to_owned(),
Some(
env::var("PUBLIC_RELEASE_CHANNEL").unwrap_or_else(|_| "canary".to_owned()),
),
),
],
..RunOptions::default()
},
)
.map(drop)
}
fn patch_macos_dev_electron_info_plist() -> Result<()> {
if !cfg!(target_os = "macos") || Path::new("/.dockerenv").exists() {
return Ok(());
}
let app_bundle = dev_electron_app_bundle_path();
let info_plist = dev_electron_info_plist_path();
if !info_plist.is_file() {
bail!(
"missing dev Electron Info.plist at {}; run `pnpm dev:desktop:install` first",
info_plist.display()
);
}
let mut changed = false;
for (key, value) in MACOS_DEV_ELECTRON_USAGE_DESCRIPTIONS {
changed |= set_or_add_plist_string(&info_plist, key, value)?;
}
if changed {
println!(
"Patched dev Electron Info.plist for macOS capture permissions: {}",
info_plist.display()
);
}
if changed || !codesign_verify(&app_bundle) {
codesign_ad_hoc(&app_bundle)?;
}
Ok(())
}
fn dev_electron_app_bundle_path() -> PathBuf {
DESKTOP_DIR.join("node_modules/electron/dist/Electron.app")
}
fn dev_electron_info_plist_path() -> PathBuf {
dev_electron_app_bundle_path().join("Contents/Info.plist")
}
fn dev_electron_binary_path() -> PathBuf {
dev_electron_app_bundle_path().join("Contents/MacOS/Electron")
}
fn set_or_add_plist_string(info_plist: &Path, key: &str, value: &str) -> Result<bool> {
if read_plist_string(info_plist, key)?.as_deref() == Some(value) {
return Ok(false);
}
if plist_key_exists(info_plist, key)? {
run_plist_buddy(info_plist, &format!("Set :{key} {value}"))?;
} else {
run_plist_buddy(info_plist, &format!("Add :{key} string {value}"))?;
}
Ok(true)
}
fn plist_key_exists(info_plist: &Path, key: &str) -> Result<bool> {
Ok(read_plist_string(info_plist, key)?.is_some())
}
fn read_plist_string(info_plist: &Path, key: &str) -> Result<Option<String>> {
let output = Command::new("/usr/libexec/PlistBuddy")
.args(["-c", &format!("Print :{key}")])
.arg(info_plist)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.with_context(|| format!("failed to read {key} from {}", info_plist.display()))?;
if !output.status.success() {
return Ok(None);
}
Ok(Some(
String::from_utf8_lossy(&output.stdout)
.trim_end()
.to_owned(),
))
}
fn run_plist_buddy(info_plist: &Path, command: &str) -> Result<()> {
let output = Command::new("/usr/libexec/PlistBuddy")
.args(["-c", command])
.arg(info_plist)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.with_context(|| format!("failed to update {}", info_plist.display()))?;
if output.status.success() {
return Ok(());
}
bail!(
"failed to update {}: {}",
info_plist.display(),
String::from_utf8_lossy(&output.stderr).trim_end()
)
}
fn codesign_verify(app_bundle: &Path) -> bool {
Command::new("codesign")
.args(["--verify", "--deep", "--strict"])
.arg(app_bundle)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|status| status.success())
.unwrap_or(false)
}
fn codesign_ad_hoc(app_bundle: &Path) -> Result<()> {
println!("Re-signing dev Electron.app after macOS permission plist patch...");
let output = Command::new("codesign")
.args(["--force", "--deep", "--sign", "-"])
.arg(app_bundle)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.with_context(|| format!("failed to re-sign {}", app_bundle.display()))?;
if output.status.success() {
return Ok(());
}
bail!(
"failed to re-sign {}: {}",
app_bundle.display(),
String::from_utf8_lossy(&output.stderr).trim_end()
)
}
fn ensure_host_macos() -> Result<()> {
if !cfg!(target_os = "macos") || Path::new("/.dockerenv").exists() {
bail!(
"`desktop canary` is a host macOS workflow. Run it on the Mac host, not inside the devcontainer."
);
}
Ok(())
}
pub async fn resolve_desktop_canary_app_url(app_url: Option<&str>) -> Result<String> {
if let Some(app_url) = app_url {
return normalize_desktop_app_url(app_url);
}
for key in [
"FLUXER_DESKTOP_CANARY_APP_URL",
"FLUXER_DESKTOP_APP_URL",
"FLUXER_PUBLIC_URL",
] {
if let Ok(value) = env::var(key) {
let value = value.trim();
if !value.is_empty()
&& !is_local_app_url(value)
&& public_app_url_is_reachable(value).await
{
return normalize_desktop_app_url(value);
}
}
}
if let Ok(public_url) = crate::tunnel::resolve_cloudflare_public_url(None)
&& public_app_url_is_reachable(&public_url).await
{
return normalize_desktop_app_url(&public_url);
}
Ok(LOCAL_APP_URL.to_owned())
}
fn normalize_desktop_app_url(raw: &str) -> Result<String> {
let url = Url::parse(raw.trim()).with_context(|| format!("invalid desktop app URL: {raw}"))?;
match url.scheme() {
"http" | "https" => Ok(url.as_str().trim_end_matches('/').to_owned()),
scheme => bail!("desktop app URL must use http or https, got {scheme}"),
}
}
fn is_local_app_url(raw: &str) -> bool {
normalize_desktop_app_url(raw)
.map(|url| matches!(url.as_str(), LOCAL_APP_URL | "http://127.0.0.1:8088"))
.unwrap_or(false)
}
async fn public_app_url_is_reachable(raw: &str) -> bool {
let Ok(base_url) = normalize_desktop_app_url(raw) else {
return false;
};
let client = match reqwest::Client::builder()
.timeout(Duration::from_secs(3))
.build()
{
Ok(client) => client,
Err(_) => return false,
};
client
.get(format!("{base_url}/gateway/_health"))
.send()
.await
.map(|response| response.status().is_success())
.unwrap_or(false)
}
fn stop_running_canary_on_host() -> Result<()> {
println!("Stopping any running {CANARY_APP_NAME} instance...");
run_best_effort(
"osascript",
&[
"-e",
&format!("tell application id \"{CANARY_BUNDLE_ID}\" to quit"),
],
);
run_best_effort(
"osascript",
&[
"-e",
&format!("tell application \"{CANARY_APP_NAME}\" to quit"),
],
);
run_best_effort("pkill", &["-TERM", "-x", CANARY_APP_NAME]);
terminate_rpc_port_processes(CANARY_RPC_PORT)?;
Ok(())
}
fn run_best_effort(program: &str, args: &[&str]) {
let _ = Command::new(program)
.args(args)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
}
fn terminate_rpc_port_processes(port: u16) -> Result<()> {
let mut pids = pids_listening_on_tcp_port(port)?;
if pids.is_empty() {
return Ok(());
}
kill_pids("-TERM", &pids)?;
let deadline = Instant::now() + Duration::from_secs(5);
while Instant::now() < deadline {
thread::sleep(Duration::from_millis(250));
pids = pids_listening_on_tcp_port(port)?;
if pids.is_empty() {
return Ok(());
}
}
kill_pids("-KILL", &pids)
}
fn pids_listening_on_tcp_port(port: u16) -> Result<Vec<String>> {
let output = Command::new("lsof")
.args(["-ti", &format!("tcp:{port}")])
.stdout(Stdio::piped())
.stderr(Stdio::null())
.output()
.context("failed to run lsof while stopping Fluxer Canary")?;
if !output.status.success() {
return Ok(Vec::new());
}
Ok(String::from_utf8_lossy(&output.stdout)
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.map(ToOwned::to_owned)
.collect())
}
fn kill_pids(signal: &str, pids: &[String]) -> Result<()> {
if pids.is_empty() {
return Ok(());
}
let status = Command::new("kill")
.arg(signal)
.args(pids)
.status()
.context("failed to run kill while stopping Fluxer Canary")?;
if status.success() {
Ok(())
} else {
bail!(
"failed to stop Fluxer Canary process(es): {}",
pids.join(", ")
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn strips_no_sandbox_only_by_container_policy() {
let args = electron_args(&["--flag".to_owned()]);
if cfg!(target_os = "linux")
&& Path::new("/.dockerenv").exists()
&& env::var("FLUXER_ELECTRON_NO_SANDBOX").as_deref() != Ok("0")
{
assert_eq!(args[0], "--no-sandbox");
}
assert!(args.contains(&"--flag".to_owned()));
}
#[test]
fn normalizes_desktop_app_urls() {
assert_eq!(
normalize_desktop_app_url("https://dev.example.test/").unwrap(),
"https://dev.example.test"
);
assert_eq!(
normalize_desktop_app_url("http://localhost:8088").unwrap(),
LOCAL_APP_URL
);
assert!(normalize_desktop_app_url("wss://dev.example.test").is_err());
}
#[test]
fn disclaimed_electron_command_re_execs_through_the_launcher() {
let command = disclaimed_electron_command(
Path::new("/tmp/fluxer-dev"),
Path::new("/tmp/Electron.app/Contents/MacOS/Electron"),
);
assert_eq!(
command,
vec![
"/tmp/fluxer-dev",
"desktop",
"exec-disclaimed",
"/tmp/Electron.app/Contents/MacOS/Electron",
"."
]
);
}
#[test]
fn dev_electron_binary_path_points_inside_the_dev_bundle() {
assert!(dev_electron_binary_path().ends_with(
"fluxer_desktop/node_modules/electron/dist/Electron.app/Contents/MacOS/Electron"
));
}
#[test]
fn dev_electron_plist_path_points_to_fluxer_desktop_electron_app() {
assert!(dev_electron_info_plist_path().ends_with(
"fluxer_desktop/node_modules/electron/dist/Electron.app/Contents/Info.plist"
));
}
#[test]
fn dev_electron_usage_descriptions_include_audio_capture() {
assert!(
MACOS_DEV_ELECTRON_USAGE_DESCRIPTIONS
.iter()
.any(|(key, _)| *key == "NSAudioCaptureUsageDescription")
);
}
}
+1048
View File
File diff suppressed because it is too large Load Diff
+81
View File
@@ -0,0 +1,81 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::{Context, Result, bail};
use std::ffi::CString;
pub fn exec_disclaimed(program: &str, args: &[String]) -> Result<()> {
let program_c = CString::new(program).context("program path contains a NUL byte")?;
let mut argv = Vec::with_capacity(args.len() + 1);
argv.push(program_c.clone());
for arg in args {
argv.push(CString::new(arg.as_str()).context("argument contains a NUL byte")?);
}
exec_disclaimed_impl(&program_c, &argv)
}
#[cfg(target_os = "macos")]
fn exec_disclaimed_impl(program: &CString, argv: &[CString]) -> Result<()> {
use std::ptr;
unsafe extern "C" {
fn responsibility_spawnattrs_setdisclaim(
attrs: *mut libc::posix_spawnattr_t,
disclaim: libc::c_int,
) -> libc::c_int;
}
let mut argv_ptrs: Vec<*mut libc::c_char> =
argv.iter().map(|arg| arg.as_ptr().cast_mut()).collect();
argv_ptrs.push(ptr::null_mut());
unsafe {
let mut attrs: libc::posix_spawnattr_t = ptr::null_mut();
let rc = libc::posix_spawnattr_init(&mut attrs);
if rc != 0 {
bail!(
"posix_spawnattr_init failed: {}",
std::io::Error::from_raw_os_error(rc)
);
}
let rc =
libc::posix_spawnattr_setflags(&mut attrs, libc::POSIX_SPAWN_SETEXEC as libc::c_short);
if rc != 0 {
libc::posix_spawnattr_destroy(&mut attrs);
bail!(
"posix_spawnattr_setflags failed: {}",
std::io::Error::from_raw_os_error(rc)
);
}
let rc = responsibility_spawnattrs_setdisclaim(&mut attrs, 1);
if rc != 0 {
libc::posix_spawnattr_destroy(&mut attrs);
bail!(
"responsibility_spawnattrs_setdisclaim failed: {}",
std::io::Error::from_raw_os_error(rc)
);
}
let mut pid: libc::pid_t = 0;
let rc = libc::posix_spawn(
&mut pid,
program.as_ptr(),
ptr::null(),
&attrs,
argv_ptrs.as_ptr(),
*libc::_NSGetEnviron(),
);
libc::posix_spawnattr_destroy(&mut attrs);
bail!(
"posix_spawn(SETEXEC) failed for {}: {}",
program.to_string_lossy(),
std::io::Error::from_raw_os_error(rc)
);
}
}
#[cfg(not(target_os = "macos"))]
fn exec_disclaimed_impl(program: &CString, _argv: &[CString]) -> Result<()> {
bail!(
"exec-disclaimed is only supported on macOS (requested program: {})",
program.to_string_lossy()
);
}
+226
View File
@@ -0,0 +1,226 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::{Context, Result, bail};
use std::collections::BTreeMap;
use std::path::Path;
pub fn read_env_file(path: &Path) -> Result<BTreeMap<String, String>> {
let mut values = BTreeMap::new();
if !path.exists() {
return Ok(values);
}
let text = std::fs::read_to_string(path)
.with_context(|| format!("failed to read {}", path.display()))?;
for (index, raw_line) in text.lines().enumerate() {
let line_number = index + 1;
let mut line = raw_line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
if let Some(rest) = line.strip_prefix("export ") {
line = rest.trim();
}
let Some((key, value)) = line.split_once('=') else {
bail!(
"Invalid env line in {}:{}: {}",
path.display(),
line_number,
raw_line
);
};
let key = key.trim();
if key.is_empty() {
bail!("Missing env key in {}:{line_number}", path.display());
}
values.insert(key.to_owned(), parse_env_value(value.trim()));
}
Ok(values)
}
pub fn read_env_files(paths: &[&Path]) -> Result<BTreeMap<String, String>> {
let mut values = BTreeMap::new();
for path in paths {
values.extend(read_env_file(path)?);
}
Ok(values)
}
pub fn merge_env_layers(layers: &[BTreeMap<String, String>]) -> BTreeMap<String, String> {
let mut merged = BTreeMap::new();
for layer in layers {
merged.extend(layer.clone());
}
merged
}
pub fn merge_default_env_with_current(
development_path: &Path,
local_path: &Path,
root_local_path: &Path,
current: BTreeMap<String, String>,
) -> Result<BTreeMap<String, String>> {
merge_default_env_with_current_and_baseline(
development_path,
local_path,
root_local_path,
current,
read_container_initial_env().unwrap_or_default(),
)
}
fn merge_default_env_with_current_and_baseline(
development_path: &Path,
local_path: &Path,
root_local_path: &Path,
current: BTreeMap<String, String>,
baseline: BTreeMap<String, String>,
) -> Result<BTreeMap<String, String>> {
let development = read_env_file(development_path)?;
let local = read_env_file(local_path)?;
let root_local = read_env_file(root_local_path)?;
let mut effective_current = current;
for (key, value) in &development {
let has_file_override = local.contains_key(key) || root_local.contains_key(key);
let current_is_development_default = effective_current.get(key) == Some(value);
let current_is_container_default = baseline
.get(key)
.is_some_and(|baseline_value| effective_current.get(key) == Some(baseline_value));
if has_file_override && (current_is_development_default || current_is_container_default) {
effective_current.remove(key);
}
}
Ok(merge_env_layers(&[
development,
local,
root_local,
effective_current,
]))
}
fn read_container_initial_env() -> Option<BTreeMap<String, String>> {
let bytes = std::fs::read("/proc/1/environ").ok()?;
let mut values = BTreeMap::new();
for entry in bytes.split(|byte| *byte == 0) {
if entry.is_empty() {
continue;
}
let text = String::from_utf8_lossy(entry);
let Some((key, value)) = text.split_once('=') else {
continue;
};
values.insert(key.to_owned(), value.to_owned());
}
Some(values)
}
fn parse_env_value(mut value: &str) -> String {
if value.len() >= 2 {
let bytes = value.as_bytes();
if (bytes[0] == b'\'' || bytes[0] == b'"') && bytes[0] == bytes[value.len() - 1] {
return value[1..value.len() - 1].to_owned();
}
}
if let Some((before, _)) = value.split_once(" #") {
value = before.trim_end();
}
value.to_owned()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_env_values_with_shell_like_comments() {
assert_eq!(parse_env_value("'quoted'"), "quoted");
assert_eq!(parse_env_value("\"quoted\""), "quoted");
assert_eq!(parse_env_value("value # comment"), "value");
assert_eq!(parse_env_value("value#not-comment"), "value#not-comment");
}
#[test]
fn reads_exports_and_reports_invalid_lines() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("env");
std::fs::write(
&file,
"\n# comment\nexport A=1\nB='two words'\nC=value # trailing\n",
)
.unwrap();
let values = read_env_file(&file).unwrap();
assert_eq!(values.get("A").unwrap(), "1");
assert_eq!(values.get("B").unwrap(), "two words");
assert_eq!(values.get("C").unwrap(), "value");
std::fs::write(&file, "nope\n").unwrap();
assert!(
read_env_file(&file)
.unwrap_err()
.to_string()
.contains("Invalid env line")
);
}
#[test]
fn local_env_overrides_injected_development_defaults() {
let dir = tempfile::tempdir().unwrap();
let development = dir.path().join("development.env");
let local = dir.path().join("local.env");
let root_local = dir.path().join(".env.local");
std::fs::write(&development, "A=default\nB=default\nC=default\n").unwrap();
std::fs::write(&local, "A=local\n").unwrap();
std::fs::write(&root_local, "B=root\n").unwrap();
let current = BTreeMap::from([
("A".to_owned(), "default".to_owned()),
("B".to_owned(), "custom".to_owned()),
("C".to_owned(), "default".to_owned()),
]);
let merged = merge_default_env_with_current_and_baseline(
&development,
&local,
&root_local,
current,
BTreeMap::new(),
)
.unwrap();
assert_eq!(merged.get("A").map(String::as_str), Some("local"));
assert_eq!(merged.get("B").map(String::as_str), Some("custom"));
assert_eq!(merged.get("C").map(String::as_str), Some("default"));
}
#[test]
fn stale_container_defaults_yield_to_file_overrides() {
let dir = tempfile::tempdir().unwrap();
let development = dir.path().join("development.env");
let local = dir.path().join("local.env");
let root_local = dir.path().join(".env.local");
std::fs::write(
&development,
"A=new-default\nB=new-default\nC=new-default\n",
)
.unwrap();
std::fs::write(&local, "A=local\nC=local\n").unwrap();
std::fs::write(&root_local, "").unwrap();
let baseline = BTreeMap::from([
("A".to_owned(), "old-default".to_owned()),
("B".to_owned(), "old-default".to_owned()),
("C".to_owned(), "old-default".to_owned()),
]);
let current = BTreeMap::from([
("A".to_owned(), "old-default".to_owned()),
("B".to_owned(), "custom".to_owned()),
("C".to_owned(), "old-default".to_owned()),
]);
let merged = merge_default_env_with_current_and_baseline(
&development,
&local,
&root_local,
current,
baseline,
)
.unwrap();
assert_eq!(merged.get("A").map(String::as_str), Some("local"));
assert_eq!(merged.get("B").map(String::as_str), Some("custom"));
assert_eq!(merged.get("C").map(String::as_str), Some("local"));
}
}
File diff suppressed because it is too large Load Diff
+420
View File
@@ -0,0 +1,420 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::gateway::{GatewayNode, gateway_dir};
use anyhow::{Context, Result, bail};
use std::collections::{BTreeMap, BTreeSet};
use std::env;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::mpsc::{Receiver, channel};
use std::time::{Duration, Instant, SystemTime};
const WATCH_POLL_INTERVAL: Duration = Duration::from_millis(1000);
const WATCH_SETTLE_INTERVAL: Duration = Duration::from_millis(300);
const RELOAD_TIMEOUT: Duration = Duration::from_secs(60);
const WATCHED_SOURCE_EXTENSIONS: &[&str] = &["erl", "hrl", "src", "rs", "toml", "config"];
pub type FileState = BTreeMap<PathBuf, (SystemTime, u64)>;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ArtifactState {
pub beams: FileState,
pub nifs: FileState,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ArtifactDiff {
pub modules: Vec<String>,
pub nifs_changed: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReloadOutcome {
pub failed_nodes: Vec<String>,
}
pub fn hot_reload_enabled() -> bool {
env::var("FLUXER_DEV_GATEWAY_HOT_RELOAD")
.map(|value| !matches!(value.to_ascii_lowercase().as_str(), "0" | "false" | "no"))
.unwrap_or(true)
}
pub fn spawn_source_watcher() -> Receiver<()> {
let (sender, receiver) = channel();
std::thread::spawn(move || {
let mut previous = scan_sources();
loop {
std::thread::sleep(WATCH_POLL_INTERVAL);
let mut current = scan_sources();
if current == previous {
continue;
}
loop {
std::thread::sleep(WATCH_SETTLE_INTERVAL);
let settled = scan_sources();
if settled == current {
break;
}
current = settled;
}
previous = current;
if sender.send(()).is_err() {
return;
}
}
});
receiver
}
fn scan_sources() -> FileState {
let dir = gateway_dir();
let mut state = FileState::new();
for root in [dir.join("src"), dir.join("include"), dir.join("native")] {
scan_tree(&mut state, &root);
}
for path in [
dir.join("rebar.config"),
dir.join("rebar.config.script"),
dir.join("rebar.lock"),
] {
record_file(&mut state, &path);
}
state
}
fn scan_tree(state: &mut FileState, root: &Path) {
let Ok(entries) = fs::read_dir(root) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
let Ok(metadata) = entry.metadata() else {
continue;
};
if metadata.is_dir() {
if !is_skipped_dir(&path) {
scan_tree(state, &path);
}
} else if is_watched_source(&path)
&& let Ok(modified) = metadata.modified()
{
state.insert(path, (modified, metadata.len()));
}
}
}
fn record_file(state: &mut FileState, path: &Path) {
if let Ok(metadata) = fs::metadata(path)
&& let Ok(modified) = metadata.modified()
{
state.insert(path.to_path_buf(), (modified, metadata.len()));
}
}
fn is_skipped_dir(path: &Path) -> bool {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name == "target" || name.starts_with('.'))
}
fn is_watched_source(path: &Path) -> bool {
path.extension()
.and_then(|extension| extension.to_str())
.is_some_and(|extension| WATCHED_SOURCE_EXTENSIONS.contains(&extension))
}
pub fn snapshot_artifacts() -> ArtifactState {
let mut state = ArtifactState::default();
let lib_root = gateway_dir().join("_build/default/lib");
if let Ok(entries) = fs::read_dir(&lib_root) {
for entry in entries.flatten() {
scan_artifact_dir(&mut state.beams, &entry.path().join("ebin"), "beam");
}
}
scan_artifact_dir(&mut state.nifs, &gateway_dir().join("priv"), "so");
state
}
fn scan_artifact_dir(state: &mut FileState, dir: &Path, extension: &str) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path
.extension()
.is_some_and(|candidate| candidate == extension)
&& let Ok(metadata) = entry.metadata()
&& let Ok(modified) = metadata.modified()
{
state.insert(path, (modified, metadata.len()));
}
}
}
pub fn changed_artifacts(before: &ArtifactState, after: &ArtifactState) -> ArtifactDiff {
let modules = after
.beams
.iter()
.filter(|(path, state)| before.beams.get(*path) != Some(*state))
.filter_map(|(path, _)| Some(path.file_stem()?.to_str()?.to_owned()))
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let nifs_changed = after
.nifs
.iter()
.any(|(path, state)| before.nifs.get(path) != Some(state));
ArtifactDiff {
modules,
nifs_changed,
}
}
pub fn build_reload_eval(nodes: &[String], modules: &[String]) -> String {
let node_list = nodes
.iter()
.map(|node| format!("'{node}'"))
.collect::<Vec<_>>()
.join(",");
let module_list = modules
.iter()
.map(|module| format!("'{module}'"))
.collect::<Vec<_>>()
.join(",");
format!(
"Nodes = [{node_list}], \
Mods = [{module_list}], \
Failed = [Node || Node <- Nodes, \
case net_adm:ping(Node) of \
pong -> \
Errors = [Mod || Mod <- Mods, \
begin \
rpc:call(Node, code, purge, [Mod], 10000), \
case rpc:call(Node, code, load_file, [Mod], 10000) of \
{{module, Mod}} -> false; \
Other -> \
io:format(\"gateway_reload_error ~s ~s ~0p~n\", [Node, Mod, Other]), \
true \
end \
end], \
Errors =/= []; \
pang -> \
io:format(\"gateway_reload_node_down ~s~n\", [Node]), \
true \
end], \
[io:format(\"gateway_reload_failed_node ~s~n\", [Failed1]) || Failed1 <- Failed], \
io:format(\"gateway_reload_done~n\"), \
halt(0)."
)
}
pub fn hot_reload_modules(
nodes: &[GatewayNode],
modules: &[String],
cookie: &str,
) -> Result<ReloadOutcome> {
assert!(!modules.is_empty());
let node_names = nodes
.iter()
.map(GatewayNode::erlang_name)
.collect::<Vec<_>>();
let eval = build_reload_eval(&node_names, modules);
let reloader_name = format!("fluxer_dev_reload_{}@127.0.0.1", std::process::id());
let mut child = Command::new("erl")
.args([
"-hidden",
"-noshell",
"-name",
&reloader_name,
"-setcookie",
cookie,
"-eval",
&eval,
])
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("failed to spawn gateway reload shell")?;
let stdout = child.stdout.take().map(spawn_reader);
let stderr = child.stderr.take().map(spawn_reader);
let status = wait_with_deadline(&mut child, RELOAD_TIMEOUT)?;
let mut output = String::new();
for handle in [stdout, stderr].into_iter().flatten() {
if let Ok(text) = handle.join() {
output.push_str(&text);
}
}
for line in output.lines().filter(|line| !line.trim().is_empty()) {
println!("[gateway:reload] {line}");
}
if !status.success() {
bail!(
"gateway reload shell exited with status {}",
status.code().unwrap_or(1)
);
}
if !output.lines().any(|line| line == "gateway_reload_done") {
bail!("gateway reload shell did not report completion");
}
Ok(parse_reload_outcome(&output))
}
pub fn parse_reload_outcome(output: &str) -> ReloadOutcome {
let failed_nodes = output
.lines()
.filter_map(|line| line.strip_prefix("gateway_reload_failed_node "))
.map(|node| node.trim().to_owned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
ReloadOutcome { failed_nodes }
}
fn spawn_reader(stream: impl Read + Send + 'static) -> std::thread::JoinHandle<String> {
std::thread::spawn(move || {
let mut reader = stream;
let mut text = String::new();
let mut bytes = Vec::new();
if reader.read_to_end(&mut bytes).is_ok() {
text = String::from_utf8_lossy(&bytes).into_owned();
}
text
})
}
fn wait_with_deadline(child: &mut Child, timeout: Duration) -> Result<std::process::ExitStatus> {
let deadline = Instant::now() + timeout;
loop {
if let Some(status) = child.try_wait()? {
return Ok(status);
}
if Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
bail!(
"gateway reload shell timed out after {}s",
timeout.as_secs()
);
}
std::thread::sleep(Duration::from_millis(100));
}
}
#[cfg(test)]
mod tests {
use super::*;
fn file_state(entries: &[(&str, u64, u64)]) -> FileState {
entries
.iter()
.map(|(path, seconds, size)| {
(
PathBuf::from(path),
(
SystemTime::UNIX_EPOCH + Duration::from_secs(*seconds),
*size,
),
)
})
.collect()
}
#[test]
fn changed_artifacts_detects_new_and_modified_beams() {
let before = ArtifactState {
beams: file_state(&[("ebin/a.beam", 1, 10), ("ebin/b.beam", 1, 20)]),
nifs: FileState::new(),
};
let after = ArtifactState {
beams: file_state(&[
("ebin/a.beam", 2, 10),
("ebin/b.beam", 1, 20),
("ebin/c.beam", 1, 30),
]),
nifs: FileState::new(),
};
let diff = changed_artifacts(&before, &after);
assert_eq!(diff.modules, vec!["a".to_owned(), "c".to_owned()]);
assert!(!diff.nifs_changed);
}
#[test]
fn changed_artifacts_ignores_unchanged_state() {
let state = ArtifactState {
beams: file_state(&[("ebin/a.beam", 1, 10)]),
nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
};
let diff = changed_artifacts(&state, &state.clone());
assert!(diff.modules.is_empty());
assert!(!diff.nifs_changed);
}
#[test]
fn changed_artifacts_flags_nif_changes() {
let before = ArtifactState {
beams: FileState::new(),
nifs: file_state(&[("priv/a_nif.so", 1, 10)]),
};
let after = ArtifactState {
beams: FileState::new(),
nifs: file_state(&[("priv/a_nif.so", 2, 11)]),
};
assert!(changed_artifacts(&before, &after).nifs_changed);
}
#[test]
fn reload_eval_quotes_nodes_and_modules() {
let eval = build_reload_eval(
&["fluxer_gateway_websocket_1@127.0.0.1".to_owned()],
&["gateway_compress".to_owned(), "push".to_owned()],
);
assert!(eval.contains("Nodes = ['fluxer_gateway_websocket_1@127.0.0.1']"));
assert!(eval.contains("Mods = ['gateway_compress','push']"));
assert!(eval.contains("halt(0)."));
}
#[test]
fn reload_outcome_parses_failed_nodes() {
let output = "gateway_reload_error n1 mod {error,nofile}\n\
gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\
gateway_reload_failed_node fluxer_gateway_guilds_2@127.0.0.1\n\
gateway_reload_done\n";
let outcome = parse_reload_outcome(output);
assert_eq!(
outcome.failed_nodes,
vec!["fluxer_gateway_guilds_2@127.0.0.1".to_owned()]
);
}
#[test]
fn reload_outcome_empty_when_no_failures() {
assert!(
parse_reload_outcome("gateway_reload_done\n")
.failed_nodes
.is_empty()
);
}
#[test]
fn watched_source_filter_accepts_known_extensions() {
assert!(is_watched_source(Path::new("src/push.erl")));
assert!(is_watched_source(Path::new("include/gateway.hrl")));
assert!(is_watched_source(Path::new("src/fluxer_gateway.app.src")));
assert!(is_watched_source(Path::new("native/a_nif/src/lib.rs")));
assert!(is_watched_source(Path::new("native/a_nif/Cargo.toml")));
assert!(!is_watched_source(Path::new("ebin/push.beam")));
assert!(!is_watched_source(Path::new("src/.push.erl.swp")));
assert!(!is_watched_source(Path::new("native/a_nif/Cargo.lock")));
}
#[test]
fn skipped_dirs_cover_build_output_and_hidden_dirs() {
assert!(is_skipped_dir(Path::new("native/a_nif/target")));
assert!(is_skipped_dir(Path::new("src/.git")));
assert!(!is_skipped_dir(Path::new("src/gateway")));
}
}
+23
View File
@@ -0,0 +1,23 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
pub mod bootstrap;
pub mod cassandra;
pub mod desktop;
pub mod dev;
pub mod disclaim;
pub mod env;
pub mod gateway;
pub mod gateway_reload;
pub mod local_k8s;
pub mod manifest;
pub mod marketing;
pub mod media_proxy;
pub mod media_stress;
pub mod native_voice_it;
pub mod paths;
pub mod proc;
pub mod proxy;
pub mod rust_services;
pub mod smoke;
pub mod tasks;
pub mod tunnel;
File diff suppressed because it is too large Load Diff
+413
View File
@@ -0,0 +1,413 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::Result;
use clap::{Args, Parser, Subcommand};
use fluxer_dev::cassandra::{
apply_schema, compute_diff, render_target_schema, verify_schema, write_diff_file,
};
use fluxer_dev::desktop::{
build_desktop, install_desktop, package_desktop, run_desktop, run_desktop_canary,
smoke_build_desktop, typecheck_desktop,
};
use fluxer_dev::env::merge_default_env_with_current;
use fluxer_dev::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL};
use fluxer_dev::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT_LOCAL_ENV_FILE};
use std::collections::BTreeMap;
use std::path::PathBuf;
#[derive(Debug, Parser)]
#[command(name = "fluxer-dev")]
struct Cli {
#[command(subcommand)]
command: Command,
}
#[derive(Debug, Subcommand)]
enum Command {
Bootstrap(BootstrapArgs),
PostStart,
Gateway(GatewayArgs),
Build,
Knip,
Test,
Typecheck,
Proxy(ProxyArgs),
Dev(DevArgs),
RustServices(RustServicesArgs),
Smoke(SmokeArgs),
Cassandra(CassandraArgs),
Desktop(DesktopArgs),
LocalK8s(LocalK8sArgs),
Marketing(MarketingArgs),
MediaProxy(MediaProxyArgs),
Tunnel(TunnelArgs),
NativeVoiceIt(fluxer_dev::native_voice_it::NativeVoiceItArgs),
}
#[derive(Debug, Args)]
struct BootstrapArgs {
#[arg(long)]
skip_install: bool,
#[arg(long)]
skip_desktop_install: bool,
}
#[derive(Debug, Args)]
struct GatewayArgs {
#[arg(value_parser = ["cluster", "single"], default_value = "cluster")]
mode: String,
}
#[derive(Debug, Args)]
struct ProxyArgs {
#[arg(long, default_value = "0.0.0.0")]
host: String,
#[arg(long, default_value_t = DEV_PROXY_PORT)]
port: u16,
}
#[derive(Debug, Args)]
struct DevArgs {
#[arg(long)]
cloudflare_tunnel: bool,
#[arg(long)]
public_url: Option<String>,
tasks: Vec<String>,
}
#[derive(Debug, Args)]
struct RustServicesArgs {
services: Vec<String>,
}
#[derive(Debug, Args)]
struct SmokeArgs {
#[arg(long)]
quick: bool,
#[arg(long)]
public: bool,
}
#[derive(Debug, Args)]
struct CassandraArgs {
#[command(subcommand)]
command: CassandraCommand,
}
#[derive(Debug, Subcommand)]
enum CassandraCommand {
Diff {
#[arg(long)]
output: Option<PathBuf>,
},
Apply,
Verify,
TargetSchema,
}
#[derive(Debug, Args)]
struct DesktopArgs {
#[command(subcommand)]
command: DesktopCommand,
}
#[derive(Debug, Subcommand)]
enum DesktopCommand {
Install,
Build {
#[arg(long)]
skip_native: bool,
},
Typecheck,
SmokeBuild,
Package {
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
builder_args: Vec<String>,
},
Run {
#[arg(long, default_value = LOCAL_APP_URL)]
app_url: String,
#[arg(long)]
no_build: bool,
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
extra_args: Vec<String>,
},
Canary {
#[arg(long)]
app_url: Option<String>,
#[arg(long)]
no_build: bool,
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
extra_args: Vec<String>,
},
#[command(hide = true)]
ExecDisclaimed {
program: String,
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
args: Vec<String>,
},
}
#[derive(Debug, Args)]
struct LocalK8sArgs {
#[command(subcommand)]
command: LocalK8sCommand,
}
#[derive(Debug, Subcommand)]
enum LocalK8sCommand {
CreateCluster,
Kubectl {
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
args: Vec<String>,
},
Helm {
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
args: Vec<String>,
},
HotpatchSmoke,
HandoffRolloutSmoke,
}
#[derive(Debug, Args)]
struct MarketingArgs {
#[command(subcommand)]
command: MarketingCommand,
}
#[derive(Debug, Subcommand)]
enum MarketingCommand {
PreprocessBlogImage(fluxer_dev::marketing::PreprocessBlogImageArgs),
PreprocessBlogVideo(fluxer_dev::marketing::PreprocessBlogVideoArgs),
}
#[derive(Debug, Args)]
struct MediaProxyArgs {
#[command(subcommand)]
command: MediaProxyCommand,
}
#[derive(Debug, Subcommand)]
#[allow(clippy::large_enum_variant)]
enum MediaProxyCommand {
BeeGifBench,
Doctor {
#[arg(long)]
repair: bool,
#[arg(long, default_value = LOCAL_APP_URL)]
base_url: String,
#[arg(long)]
path: Option<String>,
},
SeaweedfsIntegration {
#[arg(long)]
isolated_store: bool,
},
RustStressSmoke,
StressCompare(fluxer_dev::media_stress::StressCompareArgs),
SignExternalUrl(fluxer_dev::media_stress::SignExternalUrlArgs),
}
#[derive(Debug, Args)]
struct TunnelArgs {
#[command(subcommand)]
command: TunnelCommand,
}
#[derive(Debug, Subcommand)]
enum TunnelCommand {
Configure {
#[arg(long)]
public_url: String,
#[arg(long, hide_env_values = true)]
token: Option<String>,
},
PrintEnv {
#[arg(long)]
public_url: String,
},
Run {
#[arg(long, env = "FLUXER_CLOUDFLARE_TUNNEL_TOKEN", hide_env_values = true)]
token: Option<String>,
#[arg(long)]
token_file: Option<PathBuf>,
},
}
#[tokio::main]
async fn main() -> Result<()> {
if std::env::args_os()
.next()
.and_then(|path| {
std::path::PathBuf::from(path)
.file_name()
.map(|name| name.to_owned())
})
.as_deref()
== Some(std::ffi::OsStr::new("docker"))
{
std::process::exit(fluxer_dev::local_k8s::run_docker_wrapper(
std::env::args_os().skip(1),
));
}
let cli = Cli::parse();
if !matches!(
cli.command,
Command::Build | Command::Knip | Command::Test | Command::Typecheck
) {
apply_default_env()?;
}
match cli.command {
Command::Bootstrap(args) => {
fluxer_dev::bootstrap::bootstrap(args.skip_install, args.skip_desktop_install).await?;
}
Command::PostStart => fluxer_dev::bootstrap::post_start().await?,
Command::Gateway(args) if args.mode == "single" => fluxer_dev::gateway::run_gateway()?,
Command::Gateway(_) => {
std::process::exit(fluxer_dev::gateway::run_gateway_cluster().await?)
}
Command::Build => std::process::exit(fluxer_dev::tasks::run_build()?),
Command::Knip => std::process::exit(fluxer_dev::tasks::run_knip()?),
Command::Test => std::process::exit(fluxer_dev::tasks::run_test()?),
Command::Typecheck => std::process::exit(fluxer_dev::tasks::run_typecheck()?),
Command::Proxy(args) => fluxer_dev::proxy::run_proxy(&args.host, args.port).await?,
Command::Dev(args) => {
if args.cloudflare_tunnel {
fluxer_dev::tunnel::apply_cloudflare_public_url_env(args.public_url.as_deref())?;
} else if let Some(public_url) = args.public_url.as_deref() {
fluxer_dev::tunnel::apply_public_url_env(public_url)?;
}
std::process::exit(fluxer_dev::dev::run_dev(&args.tasks, args.cloudflare_tunnel).await?)
}
Command::RustServices(args) => {
std::process::exit(fluxer_dev::rust_services::run_rust_services(&args.services).await?)
}
Command::Smoke(args) => fluxer_dev::smoke::run_smoke(args.quick, args.public).await?,
Command::Cassandra(args) => match args.command {
CassandraCommand::Diff { output } => {
let diff = compute_diff(None).await?;
let output = write_diff_file(&diff, output.as_deref())?;
println!("Wrote Cassandra schema diff to {}", output.display());
if !diff.errors.is_empty() {
for error in diff.errors {
println!("error: {error}");
}
std::process::exit(1);
}
}
CassandraCommand::Apply => {
apply_schema(None).await?;
}
CassandraCommand::Verify => verify_schema(None, None).await?,
CassandraCommand::TargetSchema => print!("{}", render_target_schema("fluxer")),
},
Command::Desktop(args) => match args.command {
DesktopCommand::Install => install_desktop()?,
DesktopCommand::Build { skip_native } => build_desktop(skip_native)?,
DesktopCommand::Typecheck => typecheck_desktop()?,
DesktopCommand::SmokeBuild => smoke_build_desktop()?,
DesktopCommand::Package { builder_args } => package_desktop(&builder_args)?,
DesktopCommand::Run {
app_url,
no_build,
extra_args,
} => {
let extra_args: Vec<_> = extra_args.into_iter().filter(|arg| arg != "--").collect();
run_desktop(&app_url, &extra_args, !no_build)?;
}
DesktopCommand::Canary {
app_url,
no_build,
extra_args,
} => {
let extra_args: Vec<_> = extra_args.into_iter().filter(|arg| arg != "--").collect();
run_desktop_canary(app_url.as_deref(), &extra_args, !no_build).await?;
}
DesktopCommand::ExecDisclaimed { program, args } => {
fluxer_dev::disclaim::exec_disclaimed(&program, &args)?
}
},
Command::LocalK8s(args) => match args.command {
LocalK8sCommand::CreateCluster => fluxer_dev::local_k8s::create_cluster().await?,
LocalK8sCommand::Kubectl { args } => fluxer_dev::local_k8s::run_kubectl_cli(&args)?,
LocalK8sCommand::Helm { args } => fluxer_dev::local_k8s::run_helm_cli(&args)?,
LocalK8sCommand::HotpatchSmoke => fluxer_dev::local_k8s::run_hotpatch_smoke().await?,
LocalK8sCommand::HandoffRolloutSmoke => {
fluxer_dev::local_k8s::run_handoff_rollout_smoke().await?
}
},
Command::Marketing(args) => match args.command {
MarketingCommand::PreprocessBlogImage(args) => {
fluxer_dev::marketing::preprocess_blog_image(args)?
}
MarketingCommand::PreprocessBlogVideo(args) => {
fluxer_dev::marketing::preprocess_blog_video(args)?
}
},
Command::MediaProxy(args) => match args.command {
MediaProxyCommand::BeeGifBench => fluxer_dev::media_proxy::run_bee_gif_bench().await?,
MediaProxyCommand::Doctor {
repair,
base_url,
path,
} => {
fluxer_dev::media_proxy::run_dev_media_doctor(repair, &base_url, path.as_deref())
.await?;
}
MediaProxyCommand::SeaweedfsIntegration { isolated_store } => {
fluxer_dev::media_proxy::run_seaweedfs_media_proxy_integration(isolated_store)
.await?;
}
MediaProxyCommand::RustStressSmoke => {
fluxer_dev::media_proxy::run_rust_stress_smoke()?;
}
MediaProxyCommand::StressCompare(args) => {
std::process::exit(fluxer_dev::media_stress::run_stress_compare(args).await?)
}
MediaProxyCommand::SignExternalUrl(args) => {
println!(
"{}",
fluxer_dev::media_stress::sign_external_url(
&args.secret_key,
&args.server_url,
&args.upstream
)?
);
}
},
Command::Tunnel(args) => match args.command {
TunnelCommand::Configure { public_url, token } => {
fluxer_dev::tunnel::write_cloudflare_public_url_file(&public_url)?;
if let Some(token) = token {
fluxer_dev::tunnel::write_cloudflare_token_file(&token)?;
}
}
TunnelCommand::PrintEnv { public_url } => {
print!("{}", fluxer_dev::tunnel::public_url_env_text(&public_url)?);
}
TunnelCommand::Run { token, token_file } => std::process::exit(
fluxer_dev::tunnel::run_cloudflare_tunnel(token, token_file).await?,
),
},
Command::NativeVoiceIt(args) => fluxer_dev::native_voice_it::run(args).await?,
}
Ok(())
}
fn apply_default_env() -> Result<()> {
let current: BTreeMap<String, String> = std::env::vars().collect();
let merged = merge_default_env_with_current(
DEV_ENV_FILE.as_path(),
DEV_LOCAL_ENV_FILE.as_path(),
ROOT_LOCAL_ENV_FILE.as_path(),
current,
)?;
for (key, value) in merged {
unsafe {
std::env::set_var(key, value);
}
}
Ok(())
}
+217
View File
@@ -0,0 +1,217 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::paths::ROOT;
use std::path::PathBuf;
pub const LOOPBACK_HOST: &str = "127.0.0.1";
pub const ANY_HOST: &str = "0.0.0.0";
pub const DEV_PROXY_PORT: u16 = 8088;
pub const APP_PORT: u16 = 3000;
pub const APP_PROXY_PORT: u16 = 8773;
pub const ADMIN_PORT: u16 = 3020;
pub const API_PORT: u16 = 8080;
pub const GATEWAY_PORT: u16 = 8771;
pub const GATEWAY_WEBSOCKET_PORTS: &[u16] = &[8771, 8772, 8774];
pub const MEDIA_PROXY_PORT: u16 = 8082;
pub const MARKETING_PORT: u16 = 3010;
pub const LIVEKIT_PORT: u16 = 7880;
pub const DEVMAIL_PORT: u16 = 8025;
pub const LOCAL_APP_URL: &str = "http://localhost:8088";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProxyRoute {
pub prefix: &'static str,
pub host: &'static str,
pub port: u16,
pub strip_prefix: bool,
pub alternate_ports: &'static [u16],
}
pub const PROXY_ROUTES: &[ProxyRoute] = &[
ProxyRoute {
prefix: "/api",
host: LOOPBACK_HOST,
port: API_PORT,
strip_prefix: true,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/gateway",
host: LOOPBACK_HOST,
port: GATEWAY_PORT,
strip_prefix: true,
alternate_ports: &[8772, 8774],
},
ProxyRoute {
prefix: "/media",
host: LOOPBACK_HOST,
port: MEDIA_PROXY_PORT,
strip_prefix: true,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/fluxer-uploads",
host: LOOPBACK_HOST,
port: 8333,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/livekit",
host: "livekit",
port: LIVEKIT_PORT,
strip_prefix: true,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/devmail",
host: "mailpit",
port: DEVMAIL_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/admin",
host: LOOPBACK_HOST,
port: ADMIN_PORT,
strip_prefix: true,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/marketing/branding",
host: LOOPBACK_HOST,
port: APP_PROXY_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/marketing/flags",
host: LOOPBACK_HOST,
port: APP_PROXY_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/marketing/pwa-install",
host: LOOPBACK_HOST,
port: APP_PROXY_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/marketing/screenshots",
host: LOOPBACK_HOST,
port: APP_PROXY_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/marketing",
host: LOOPBACK_HOST,
port: MARKETING_PORT,
strip_prefix: true,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/assets",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/lazy-compilation-using-",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/manifest.json",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/browserconfig.xml",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/robots.txt",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/sw.js",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/sw.js.map",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/version.json",
host: LOOPBACK_HOST,
port: APP_PORT,
strip_prefix: false,
alternate_ports: &[],
},
ProxyRoute {
prefix: "/",
host: LOOPBACK_HOST,
port: APP_PROXY_PORT,
strip_prefix: false,
alternate_ports: &[],
},
];
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RustServiceSpec {
pub name: &'static str,
pub package: &'static str,
pub path: PathBuf,
pub port_base: u16,
}
pub fn rust_services() -> Vec<RustServiceSpec> {
vec![
RustServiceSpec {
name: "messages",
package: "fluxer-messages",
path: ROOT.join("fluxer_messages"),
port_base: 8112,
},
RustServiceSpec {
name: "snowflakes",
package: "fluxer-snowflakes",
path: ROOT.join("fluxer_snowflakes"),
port_base: 8120,
},
RustServiceSpec {
name: "unfurl",
package: "fluxer-unfurl",
path: ROOT.join("fluxer_unfurl"),
port_base: 8122,
},
RustServiceSpec {
name: "users",
package: "fluxer-users",
path: ROOT.join("fluxer_users"),
port_base: 8124,
},
]
}
+481
View File
@@ -0,0 +1,481 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::paths::{ROOT, which};
use crate::proc::format_command;
use anyhow::{Context, Result, bail};
use clap::{Args, ValueEnum};
use std::collections::BTreeSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
const DEFAULT_BLOG_OUTPUT_DIR: &str = "fluxer_marketing/static/blog";
#[derive(Debug, Clone, Args)]
pub struct PreprocessBlogImageArgs {
#[arg(long)]
input: PathBuf,
#[arg(long)]
name: String,
#[arg(long, default_value = DEFAULT_BLOG_OUTPUT_DIR)]
output_dir: PathBuf,
#[arg(long, value_enum, default_value_t = ImageFallback::Png)]
fallback: ImageFallback,
#[arg(long, value_delimiter = ' ', default_values_t = [640, 960, 1280, 2000])]
widths: Vec<u32>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum ImageFallback {
Png,
Jpg,
}
#[derive(Debug, Clone, Args)]
pub struct PreprocessBlogVideoArgs {
#[arg(long)]
input: PathBuf,
#[arg(long)]
name: String,
#[arg(long, default_value = DEFAULT_BLOG_OUTPUT_DIR)]
output_dir: PathBuf,
#[arg(long, default_value_t = 640)]
max_width: u32,
#[arg(long)]
fps: Option<String>,
#[arg(long, default_value = "0.5")]
poster_time: String,
#[arg(long, value_enum, default_value_t = VideoAudio::Keep)]
audio: VideoAudio,
#[arg(long, default_value_t = 24)]
mp4_crf: u32,
#[arg(long, default_value_t = 34)]
webm_crf: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum VideoAudio {
Keep,
None,
}
pub fn preprocess_blog_image(args: PreprocessBlogImageArgs) -> Result<()> {
require_tools(&["magick", "cwebp", "ffmpeg"])?;
validate_asset_name(&args.name)?;
ensure_input_file(&args.input)?;
if args.widths.is_empty() {
bail!("--widths must include at least one width");
}
let (original_width, _) = image::image_dimensions(&args.input)
.with_context(|| format!("could not read image dimensions: {}", args.input.display()))?;
if original_width == 0 {
bail!("could not read image width: {}", args.input.display());
}
fs::create_dir_all(&args.output_dir)?;
for width in target_widths(original_width, &args.widths)? {
let base = args.output_dir.join(format!("{}-{width}", args.name));
let fallback_file = match args.fallback {
ImageFallback::Png => {
let output = base.with_extension("png");
run(&image_magick_png_command(&args.input, width, &output)?)?;
output
}
ImageFallback::Jpg => {
let output = base.with_extension("jpg");
run(&image_magick_jpg_command(&args.input, width, &output)?)?;
output
}
};
run(&cwebp_command(
&fallback_file,
&base.with_extension("webp"),
)?)?;
run(&avif_still_command(
&fallback_file,
&base.with_extension("avif"),
)?)?;
}
Ok(())
}
pub fn preprocess_blog_video(args: PreprocessBlogVideoArgs) -> Result<()> {
require_tools(&["ffmpeg", "ffprobe"])?;
validate_asset_name(&args.name)?;
ensure_input_file(&args.input)?;
if args.max_width < 2 {
bail!("--max-width must be an integer greater than 1");
}
ensure_video_stream(&args.input)?;
fs::create_dir_all(&args.output_dir)?;
let base = args.output_dir.join(&args.name);
let profile = VideoProfile::from_args(&args);
run(&mp4_command(
&args.input,
&base.with_extension("mp4"),
&profile,
)?)?;
run(&webm_command(
&args.input,
&base.with_extension("webm"),
&profile,
)?)?;
run(&poster_command(
&args.input,
&base.with_file_name(format!("{}-poster.jpg", args.name)),
&profile,
)?)?;
Ok(())
}
fn require_tools(tools: &[&str]) -> Result<()> {
for tool in tools {
if which(tool).is_none() {
bail!("missing required tool: {tool}");
}
}
Ok(())
}
fn validate_asset_name(name: &str) -> Result<()> {
if name.trim().is_empty() {
bail!("--name must not be empty");
}
if name.contains('/') || name.contains('\\') {
bail!("--name must be a file name, not a path");
}
Ok(())
}
fn ensure_input_file(input: &Path) -> Result<()> {
if !input.is_file() {
bail!("input file not found: {}", input.display());
}
Ok(())
}
fn ensure_video_stream(input: &Path) -> Result<()> {
let status = Command::new("ffprobe")
.args([
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0",
])
.arg(input)
.status()
.with_context(|| format!("failed to run ffprobe for {}", input.display()))?;
if !status.success() {
bail!("could not read video stream: {}", input.display());
}
Ok(())
}
fn target_widths(original_width: u32, requested_widths: &[u32]) -> Result<Vec<u32>> {
let mut seen = BTreeSet::new();
let mut widths = Vec::new();
for requested in requested_widths {
if *requested == 0 {
bail!("invalid width: {requested}");
}
let width = (*requested).min(original_width);
if seen.insert(width) {
widths.push(width);
}
}
Ok(widths)
}
fn image_magick_png_command(input: &Path, width: u32, output: &Path) -> Result<Vec<String>> {
Ok(vec![
"magick".to_owned(),
path_arg(input),
"-auto-orient".to_owned(),
"-resize".to_owned(),
format!("{width}x>"),
"-strip".to_owned(),
"-alpha".to_owned(),
"off".to_owned(),
format!("PNG24:{}", path_arg(output)),
])
}
fn image_magick_jpg_command(input: &Path, width: u32, output: &Path) -> Result<Vec<String>> {
Ok(vec![
"magick".to_owned(),
path_arg(input),
"-auto-orient".to_owned(),
"-resize".to_owned(),
format!("{width}x>"),
"-strip".to_owned(),
"-alpha".to_owned(),
"remove".to_owned(),
"-background".to_owned(),
"white".to_owned(),
"-quality".to_owned(),
"84".to_owned(),
"-interlace".to_owned(),
"Plane".to_owned(),
path_arg(output),
])
}
fn cwebp_command(input: &Path, output: &Path) -> Result<Vec<String>> {
Ok(vec![
"cwebp".to_owned(),
"-quiet".to_owned(),
"-q".to_owned(),
"82".to_owned(),
"-m".to_owned(),
"6".to_owned(),
"-af".to_owned(),
path_arg(input),
"-o".to_owned(),
path_arg(output),
])
}
fn avif_still_command(input: &Path, output: &Path) -> Result<Vec<String>> {
Ok(vec![
"ffmpeg".to_owned(),
"-hide_banner".to_owned(),
"-loglevel".to_owned(),
"error".to_owned(),
"-y".to_owned(),
"-i".to_owned(),
path_arg(input),
"-frames:v".to_owned(),
"1".to_owned(),
"-c:v".to_owned(),
"libaom-av1".to_owned(),
"-still-picture".to_owned(),
"1".to_owned(),
"-crf".to_owned(),
"34".to_owned(),
"-cpu-used".to_owned(),
"6".to_owned(),
path_arg(output),
])
}
#[derive(Debug, Clone)]
struct VideoProfile {
scale_filter: String,
video_filter: String,
audio_map: Vec<String>,
mp4_audio: Vec<String>,
webm_audio: Vec<String>,
mp4_crf: String,
webm_crf: String,
poster_time: String,
}
impl VideoProfile {
fn from_args(args: &PreprocessBlogVideoArgs) -> Self {
let scale_filter = format!("scale='trunc(min({},iw)/2)*2':-2", args.max_width);
let video_filter = match &args.fps {
Some(fps) if !fps.is_empty() => format!("{scale_filter},fps={fps},format=yuv420p"),
_ => format!("{scale_filter},format=yuv420p"),
};
let (audio_map, mp4_audio, webm_audio) = match args.audio {
VideoAudio::Keep => (
strings(&["-map", "0:v:0", "-map", "0:a?"]),
strings(&["-c:a", "aac", "-b:a", "128k"]),
strings(&["-c:a", "libopus", "-b:a", "96k"]),
),
VideoAudio::None => (
strings(&["-map", "0:v:0"]),
strings(&["-an"]),
strings(&["-an"]),
),
};
Self {
scale_filter,
video_filter,
audio_map,
mp4_audio,
webm_audio,
mp4_crf: args.mp4_crf.to_string(),
webm_crf: args.webm_crf.to_string(),
poster_time: args.poster_time.clone(),
}
}
}
fn mp4_command(input: &Path, output: &Path, profile: &VideoProfile) -> Result<Vec<String>> {
let mut command = strings(&["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i"]);
command.push(path_arg(input));
command.extend(profile.audio_map.iter().cloned());
command.extend(strings(&[
"-vf",
&profile.video_filter,
"-c:v",
"libx264",
"-preset",
"slow",
"-crf",
&profile.mp4_crf,
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
]));
command.extend(profile.mp4_audio.iter().cloned());
command.push(path_arg(output));
Ok(command)
}
fn webm_command(input: &Path, output: &Path, profile: &VideoProfile) -> Result<Vec<String>> {
let mut command = strings(&["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i"]);
command.push(path_arg(input));
command.extend(profile.audio_map.iter().cloned());
command.extend(strings(&[
"-vf",
&profile.video_filter,
"-c:v",
"libvpx-vp9",
"-b:v",
"0",
"-crf",
&profile.webm_crf,
"-deadline",
"good",
"-cpu-used",
"4",
"-pix_fmt",
"yuv420p",
]));
command.extend(profile.webm_audio.iter().cloned());
command.push(path_arg(output));
Ok(command)
}
fn poster_command(input: &Path, output: &Path, profile: &VideoProfile) -> Result<Vec<String>> {
Ok(vec![
"ffmpeg".to_owned(),
"-hide_banner".to_owned(),
"-loglevel".to_owned(),
"error".to_owned(),
"-y".to_owned(),
"-ss".to_owned(),
profile.poster_time.clone(),
"-i".to_owned(),
path_arg(input),
"-map".to_owned(),
"0:v:0".to_owned(),
"-frames:v".to_owned(),
"1".to_owned(),
"-vf".to_owned(),
profile.scale_filter.clone(),
"-q:v".to_owned(),
"3".to_owned(),
path_arg(output),
])
}
fn run(args: &[String]) -> Result<()> {
println!("$ {}", format_command(args));
let status = Command::new(&args[0])
.args(&args[1..])
.current_dir(ROOT.as_path())
.status()
.with_context(|| format!("failed to run {}", format_command(args)))?;
if !status.success() {
bail!(
"command failed with exit code {}: {}",
status.code().unwrap_or(1),
format_command(args)
);
}
Ok(())
}
fn strings(values: &[&str]) -> Vec<String> {
values.iter().map(|value| (*value).to_owned()).collect()
}
fn path_arg(path: &Path) -> String {
path.display().to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn target_widths_clamp_and_deduplicate_in_request_order() {
assert_eq!(
target_widths(1000, &[640, 960, 1280, 1000, 960]).unwrap(),
vec![640, 960, 1000]
);
assert!(target_widths(1000, &[0]).is_err());
}
#[test]
fn builds_png_fallback_command() {
let command =
image_magick_png_command(Path::new("in.png"), 640, Path::new("out.png")).unwrap();
assert_eq!(command[0], "magick");
assert!(command.contains(&"640x>".to_owned()));
assert_eq!(command.last().unwrap(), "PNG24:out.png");
}
#[test]
fn builds_jpg_fallback_command() {
let command =
image_magick_jpg_command(Path::new("in.png"), 640, Path::new("out.jpg")).unwrap();
assert!(command.contains(&"-background".to_owned()));
assert!(command.contains(&"white".to_owned()));
assert_eq!(command.last().unwrap(), "out.jpg");
}
#[test]
fn builds_video_profile_with_fps_and_no_audio() {
let profile = VideoProfile::from_args(&PreprocessBlogVideoArgs {
input: PathBuf::from("in.gif"),
name: "clip".to_owned(),
output_dir: PathBuf::from("out"),
max_width: 640,
fps: Some("15".to_owned()),
poster_time: "0.5".to_owned(),
audio: VideoAudio::None,
mp4_crf: 24,
webm_crf: 34,
});
assert_eq!(
profile.video_filter,
"scale='trunc(min(640,iw)/2)*2':-2,fps=15,format=yuv420p"
);
assert_eq!(profile.audio_map, strings(&["-map", "0:v:0"]));
assert_eq!(profile.mp4_audio, strings(&["-an"]));
}
#[test]
fn builds_video_encode_commands() {
let profile = VideoProfile::from_args(&PreprocessBlogVideoArgs {
input: PathBuf::from("in.gif"),
name: "clip".to_owned(),
output_dir: PathBuf::from("out"),
max_width: 320,
fps: None,
poster_time: "1.25".to_owned(),
audio: VideoAudio::Keep,
mp4_crf: 20,
webm_crf: 30,
});
let mp4 = mp4_command(Path::new("in.gif"), Path::new("clip.mp4"), &profile).unwrap();
assert!(mp4.contains(&"libx264".to_owned()));
assert!(mp4.contains(&"+faststart".to_owned()));
assert_eq!(mp4.last().unwrap(), "clip.mp4");
let poster =
poster_command(Path::new("in.gif"), Path::new("clip-poster.jpg"), &profile).unwrap();
assert!(poster.contains(&"1.25".to_owned()));
assert_eq!(poster.last().unwrap(), "clip-poster.jpg");
}
}
File diff suppressed because it is too large Load Diff
+801
View File
@@ -0,0 +1,801 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::{Context, Result, bail};
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
use clap::Args;
use hmac::{Hmac, KeyInit, Mac};
use image::{AnimationDecoder, RgbaImage, codecs::gif::GifDecoder, imageops::FilterType};
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use std::collections::BTreeMap;
use std::io::{BufReader, Cursor};
use std::process::Command;
use std::time::{Duration, Instant};
use tokio::task::JoinSet;
const DEFAULT_PREFIXES: &[&str] = &[
"attachments/",
"emojis/",
"stickers/",
"avatars/",
"icons/",
"banners/",
];
const KNOWN_EXTS: &[&str] = &[
"png", "jpg", "jpeg", "webp", "gif", "apng", "avif", "heic", "heif", "jxl",
];
#[derive(Debug, Clone, Args)]
pub struct StressCompareArgs {
#[arg(long, help = "v1 base URL, for example http://media-proxy:8080")]
pub v1: String,
#[arg(long, help = "v2 base URL, for example http://10.244.147.209:8080")]
pub v2: String,
#[arg(long, default_value = "fluxer")]
pub bucket: String,
#[arg(
long,
env = "FLUXER_S3_ENDPOINT",
default_value = "https://ewr1.vultrobjects.com"
)]
pub endpoint: String,
#[arg(long, env = "FLUXER_S3_REGION", default_value = "ewr1")]
pub region: String,
#[arg(long, env = "FLUXER_S3_ACCESS_KEY_ID")]
pub access_key: Option<String>,
#[arg(long, env = "FLUXER_S3_SECRET_ACCESS_KEY")]
pub secret_key: Option<String>,
#[arg(long, action = clap::ArgAction::Append)]
pub prefix: Vec<String>,
#[arg(long, default_value_t = 20)]
pub per_prefix: usize,
#[arg(long, default_value = "plain,resize,format,resize+format")]
pub matrix: String,
#[arg(long, default_value_t = 256)]
pub size: u32,
#[arg(long, default_value = "webp")]
pub format: String,
#[arg(long, default_value_t = 8)]
pub concurrency: usize,
#[arg(long, default_value_t = 30.0)]
pub timeout: f64,
#[arg(long, default_value_t = 42)]
pub seed: u64,
#[arg(long, default_value = "-", help = "JSON report path, or - for stdout")]
pub report: String,
#[arg(long, default_value_t = 40)]
pub max_issues_print: usize,
}
#[derive(Debug, Clone, Args)]
pub struct SignExternalUrlArgs {
#[arg(long)]
pub secret_key: String,
#[arg(long)]
pub server_url: String,
pub upstream: String,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct StressCase {
pub label: String,
pub url_path: String,
#[serde(skip_serializing_if = "String::is_empty")]
pub query: String,
pub expect_image: bool,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct StressResult {
pub case: StressCase,
pub v1_status: i32,
pub v2_status: i32,
pub v1_ct: String,
pub v2_ct: String,
pub v1_size: usize,
pub v2_size: usize,
pub issues: Vec<String>,
pub elapsed_v1_ms: f64,
pub elapsed_v2_ms: f64,
}
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
pub struct PrefixSummary {
pub ok: u64,
pub fail: u64,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct StressSummary {
pub total: usize,
pub issues: usize,
pub by_prefix: BTreeMap<String, PrefixSummary>,
pub issues_sample: Vec<StressResult>,
}
#[derive(Debug)]
struct FetchResult {
status: i32,
content_type: String,
body: Vec<u8>,
elapsed_ms: f64,
}
#[derive(Debug, Deserialize)]
struct AwsListObjects {
#[serde(rename = "Contents", default)]
contents: Vec<AwsObject>,
#[serde(rename = "IsTruncated", default)]
is_truncated: bool,
#[serde(rename = "NextContinuationToken")]
next_continuation_token: Option<String>,
}
#[derive(Debug, Deserialize)]
struct AwsObject {
#[serde(rename = "Key")]
key: String,
}
#[derive(Debug, Clone)]
struct StableRng(u64);
impl StableRng {
fn new(seed: u64) -> Self {
Self(seed)
}
fn next_below(&mut self, upper: usize) -> usize {
self.0 = self
.0
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
((self.0 >> 32) as usize) % upper
}
}
pub async fn run_stress_compare(args: StressCompareArgs) -> Result<i32> {
let access_key = args
.access_key
.as_deref()
.filter(|value| !value.is_empty())
.context("missing S3 creds: pass --access-key or set FLUXER_S3_ACCESS_KEY_ID")?;
let secret_key = args
.secret_key
.as_deref()
.filter(|value| !value.is_empty())
.context("missing S3 creds: pass --secret-key or set FLUXER_S3_SECRET_ACCESS_KEY")?;
let prefixes = if args.prefix.is_empty() {
DEFAULT_PREFIXES
.iter()
.map(|value| (*value).to_owned())
.collect()
} else {
args.prefix.clone()
};
let mut rng = StableRng::new(args.seed);
let mut cases = Vec::new();
for prefix in &prefixes {
let keys = list_random_keys(
&RandomKeyListRequest {
endpoint: &args.endpoint,
access_key,
secret_key,
region: &args.region,
bucket: &args.bucket,
prefix,
want: args.per_prefix,
},
&mut rng,
)?;
for key in keys {
cases.extend(cases_for_key(&key, &args.matrix, args.size, &args.format));
}
}
eprintln!("# {} cases across {} prefixes", cases.len(), prefixes.len());
let results = compare_cases(
cases,
&args.v1,
&args.v2,
args.timeout,
args.concurrency.max(1),
)
.await?;
let summary = summarize_results(&results, args.max_issues_print);
let output = serde_json::to_string_pretty(&summary)?;
if args.report == "-" {
println!("{output}");
} else {
std::fs::write(&args.report, output)
.with_context(|| format!("failed to write {}", args.report))?;
}
Ok(summary.issues.min(255) as i32)
}
pub fn sign_external_url(secret_key: &str, server_url: &str, upstream: &str) -> Result<String> {
let path = format!("v2/{}", URL_SAFE_NO_PAD.encode(upstream.as_bytes()));
let mut mac = Hmac::<Sha256>::new_from_slice(secret_key.as_bytes())
.context("failed to create HMAC signer")?;
mac.update(path.as_bytes());
let signature = URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes());
Ok(format!(
"{}/external/{}/{}",
server_url.trim_end_matches('/'),
signature,
path
))
}
struct RandomKeyListRequest<'a> {
endpoint: &'a str,
access_key: &'a str,
secret_key: &'a str,
region: &'a str,
bucket: &'a str,
prefix: &'a str,
want: usize,
}
fn list_random_keys(
request: &RandomKeyListRequest<'_>,
rng: &mut StableRng,
) -> Result<Vec<String>> {
if request.want == 0 {
return Ok(Vec::new());
}
let mut keys = Vec::new();
let mut seen = 0usize;
let mut continuation_token = None;
let scan_limit = request.want.saturating_mul(50).max(5000);
loop {
let page = list_objects_page(
request.endpoint,
request.access_key,
request.secret_key,
request.region,
request.bucket,
request.prefix,
continuation_token.as_deref(),
)?;
for object in page.contents {
seen += 1;
if keys.len() < request.want {
keys.push(object.key);
} else {
let index = rng.next_below(seen);
if index < request.want {
keys[index] = object.key;
}
}
}
if seen >= scan_limit || !page.is_truncated {
break;
}
continuation_token = page.next_continuation_token;
if continuation_token.is_none() {
break;
}
}
Ok(keys)
}
fn list_objects_page(
endpoint: &str,
access_key: &str,
secret_key: &str,
region: &str,
bucket: &str,
prefix: &str,
continuation_token: Option<&str>,
) -> Result<AwsListObjects> {
let mut command = Command::new("aws");
command
.arg("--no-cli-pager")
.arg("--endpoint-url")
.arg(endpoint)
.arg("--region")
.arg(region)
.arg("s3api")
.arg("list-objects-v2")
.arg("--bucket")
.arg(bucket)
.arg("--prefix")
.arg(prefix)
.arg("--max-keys")
.arg("1000")
.arg("--output")
.arg("json")
.env("AWS_ACCESS_KEY_ID", access_key)
.env("AWS_SECRET_ACCESS_KEY", secret_key)
.env("AWS_DEFAULT_REGION", region)
.env("AWS_EC2_METADATA_DISABLED", "true");
if let Some(token) = continuation_token {
command.arg("--continuation-token").arg(token);
}
let output = command.output().context("failed to run aws s3api")?;
if !output.status.success() {
bail!(
"aws s3api list-objects-v2 failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
);
}
serde_json::from_slice(&output.stdout).context("failed to parse aws s3api JSON")
}
pub fn cases_for_key(key: &str, matrix: &str, size: u32, format: &str) -> Vec<StressCase> {
let Some(url_path) = url_for_key(key) else {
return Vec::new();
};
let is_attachment = key.starts_with("attachments/");
let mut cases = Vec::new();
for variant in matrix.split(',').map(str::trim).filter(|v| !v.is_empty()) {
match variant {
"plain" => cases.push(StressCase {
label: format!("{key}|plain"),
url_path: url_path.clone(),
query: String::new(),
expect_image: true,
}),
"resize" => cases.push(StressCase {
label: format!("{key}|resize{size}"),
url_path: url_path.clone(),
query: resize_query(is_attachment, size),
expect_image: true,
}),
"format" => cases.push(StressCase {
label: format!("{key}|fmt={format}"),
url_path: url_path.clone(),
query: format!("format={format}"),
expect_image: true,
}),
"resize+format" if is_attachment => cases.push(StressCase {
label: format!("{key}|w{size}+{format}"),
url_path: url_path.clone(),
query: format!("format={format}&width={size}&height={size}"),
expect_image: true,
}),
"resize+format" => cases.push(StressCase {
label: format!("{key}|s{size}+{format}"),
url_path: url_path.clone(),
query: format!("size={size}&format={format}"),
expect_image: true,
}),
_ => {}
}
}
cases
}
pub fn url_for_key(key: &str) -> Option<String> {
let parts = key.split('/').collect::<Vec<_>>();
let prefix = parts.first().copied()?;
match prefix {
"emojis" | "stickers" if parts.len() == 2 => Some(format!(
"/{prefix}/{}",
quote_component(&ensure_ext(parts[1], "webp"))
)),
"attachments" if parts.len() == 4 => Some(format!(
"/attachments/{}/{}/{}",
quote_component(parts[1]),
quote_component(parts[2]),
quote_component(parts[3])
)),
"avatars" | "icons" | "banners" | "splashes" | "embed-splashes" if parts.len() == 3 => {
Some(format!(
"/{prefix}/{}/{}",
quote_component(parts[1]),
quote_component(&ensure_ext(parts[2], "webp"))
))
}
_ => None,
}
}
fn ensure_ext(name: &str, default: &str) -> String {
let lower = name.to_ascii_lowercase();
if KNOWN_EXTS
.iter()
.any(|extension| lower.ends_with(&format!(".{extension}")))
{
name.to_owned()
} else {
format!("{name}.{default}")
}
}
fn quote_component(value: &str) -> String {
urlencoding::encode(value).into_owned()
}
fn resize_query(is_attachment: bool, size: u32) -> String {
if is_attachment {
format!("width={size}&height={size}")
} else {
format!("size={size}")
}
}
async fn compare_cases(
cases: Vec<StressCase>,
v1_base: &str,
v2_base: &str,
timeout: f64,
concurrency: usize,
) -> Result<Vec<StressResult>> {
let client = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.timeout(Duration::from_secs_f64(timeout.max(0.1)))
.build()?;
let mut pending = cases.into_iter();
let mut join_set = JoinSet::new();
let mut results = Vec::new();
let mut submitted = 0usize;
loop {
while join_set.len() < concurrency {
let Some(case) = pending.next() else {
break;
};
let client = client.clone();
let v1_base = v1_base.to_owned();
let v2_base = v2_base.to_owned();
join_set.spawn(async move { compare_one(client, case, v1_base, v2_base).await });
submitted += 1;
}
let Some(joined) = join_set.join_next().await else {
break;
};
let result = joined.context("stress compare worker panicked")?;
results.push(result);
if results.len() % 25 == 0 {
let issue_count = results
.iter()
.filter(|result| !result.issues.is_empty())
.count();
eprintln!(
"# {}/{} done, {} issues so far",
results.len(),
submitted + pending.len(),
issue_count
);
}
}
Ok(results)
}
async fn compare_one(
client: reqwest::Client,
case: StressCase,
v1_base: String,
v2_base: String,
) -> StressResult {
let v1 = fetch(&client, &v1_base, &case.url_path, &case.query).await;
let v2 = fetch(&client, &v2_base, &case.url_path, &case.query).await;
let mut result = StressResult {
case,
v1_status: v1.status,
v2_status: v2.status,
v1_ct: v1.content_type,
v2_ct: v2.content_type,
v1_size: v1.body.len(),
v2_size: v2.body.len(),
issues: Vec::new(),
elapsed_v1_ms: v1.elapsed_ms,
elapsed_v2_ms: v2.elapsed_ms,
};
if result.v1_status != result.v2_status {
result.issues.push(format!(
"status mismatch v1={} v2={}",
result.v1_status, result.v2_status
));
return result;
}
if result.v1_status >= 400 {
return result;
}
if !result.case.expect_image {
if v1.body != v2.body {
result.issues.push(format!(
"body diverges v1={}B v2={}B",
result.v1_size, result.v2_size
));
}
return result;
}
let (a_image, a_frames, a_error) = decode_image(&v1.body);
let (b_image, b_frames, b_error) = decode_image(&v2.body);
match (a_error.as_deref(), b_error.as_deref()) {
(Some(a_error), None) => result
.issues
.push(format!("v1 decode failed but v2 ok: {a_error}")),
(None, Some(b_error)) => result.issues.push(format!(
"v2 decode failed: {b_error} (v1 ok, {a_frames} frame(s))"
)),
(Some(_), Some(_)) => {
if result.v1_size.abs_diff(result.v2_size) > result.v1_size.saturating_div(4).max(256) {
result.issues.push(format!(
"both decode-failed, size differs ({} vs {})",
result.v1_size, result.v2_size
));
}
return result;
}
(None, None) => {}
}
if let (Some(a_image), Some(b_image)) = (a_image, b_image) {
if a_frames != b_frames {
result
.issues
.push(format!("frame count v1={a_frames} v2={b_frames}"));
}
let similarity = pixel_similarity(&a_image, &b_image);
if similarity > 0.08 {
result.issues.push(format!(
"pixel diff {similarity:.3} (v1 size {}x{} v2 size {}x{})",
a_image.width(),
a_image.height(),
b_image.width(),
b_image.height()
));
}
}
result
}
async fn fetch(client: &reqwest::Client, base: &str, path: &str, query: &str) -> FetchResult {
let url = format!(
"{}{}{}",
base.trim_end_matches('/'),
path,
if query.is_empty() {
String::new()
} else {
format!("?{query}")
}
);
let started = Instant::now();
match client.get(url).send().await {
Ok(response) => {
let status = response.status().as_u16() as i32;
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
match response.bytes().await {
Ok(body) => FetchResult {
status,
content_type,
body: body.to_vec(),
elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
},
Err(error) => FetchResult {
status: -1,
content_type: format!("<error: {}: {error}>", error_kind(&error)),
body: Vec::new(),
elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
},
}
}
Err(error) => FetchResult {
status: -1,
content_type: format!("<error: {}: {error}>", error_kind(&error)),
body: Vec::new(),
elapsed_ms: started.elapsed().as_secs_f64() * 1000.0,
},
}
}
fn error_kind(error: &reqwest::Error) -> &'static str {
if error.is_timeout() {
"Timeout"
} else if error.is_connect() {
"Connect"
} else if error.is_decode() {
"Decode"
} else {
"Request"
}
}
fn decode_image(data: &[u8]) -> (Option<RgbaImage>, usize, Option<String>) {
if data.starts_with(b"GIF87a") || data.starts_with(b"GIF89a") {
return decode_gif(data);
}
match image::load_from_memory(data) {
Ok(image) => (Some(image.to_rgba8()), 1, None),
Err(error) => (None, 0, Some(error.to_string())),
}
}
fn decode_gif(data: &[u8]) -> (Option<RgbaImage>, usize, Option<String>) {
let reader = BufReader::new(Cursor::new(data));
let decoder = match GifDecoder::new(reader) {
Ok(decoder) => decoder,
Err(error) => return (None, 0, Some(error.to_string())),
};
match decoder.into_frames().collect_frames() {
Ok(frames) => {
let first = frames.first().map(|frame| frame.buffer().clone());
(first, frames.len(), None)
}
Err(error) => (None, 0, Some(error.to_string())),
}
}
fn pixel_similarity(a: &RgbaImage, b: &RgbaImage) -> f64 {
let b_resized;
let b = if a.dimensions() == b.dimensions() {
b
} else {
let width_diff = a.width().abs_diff(b.width());
let height_diff = a.height().abs_diff(b.height());
if width_diff > 8 || height_diff > 8 {
return 1.0;
}
b_resized = image::imageops::resize(b, a.width(), a.height(), FilterType::Lanczos3);
&b_resized
};
let sum = a
.as_raw()
.iter()
.zip(b.as_raw())
.map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs() as u64)
.sum::<u64>();
sum as f64 / (a.as_raw().len() as f64 * 255.0)
}
pub fn summarize_results(results: &[StressResult], max_issues_print: usize) -> StressSummary {
let mut by_prefix = BTreeMap::new();
for result in results {
let prefix = result
.case
.url_path
.split('/')
.nth(1)
.unwrap_or("")
.to_owned();
let entry = by_prefix
.entry(prefix)
.or_insert(PrefixSummary { ok: 0, fail: 0 });
if result.issues.is_empty() {
entry.ok += 1;
} else {
entry.fail += 1;
}
}
let issues_sample = results
.iter()
.filter(|result| !result.issues.is_empty())
.take(max_issues_print)
.cloned()
.collect::<Vec<_>>();
StressSummary {
total: results.len(),
issues: results
.iter()
.filter(|result| !result.issues.is_empty())
.count(),
by_prefix,
issues_sample,
}
}
#[cfg(test)]
mod tests {
use super::*;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
#[test]
fn maps_s3_keys_to_public_routes() {
assert_eq!(
url_for_key("avatars/42/hash"),
Some("/avatars/42/hash.webp".to_owned())
);
assert_eq!(
url_for_key("avatars/42/hash.avif"),
Some("/avatars/42/hash.avif".to_owned())
);
assert_eq!(
url_for_key("emojis/123"),
Some("/emojis/123.webp".to_owned())
);
assert_eq!(
url_for_key("attachments/a b/c/name#.png"),
Some("/attachments/a%20b/c/name%23.png".to_owned())
);
assert_eq!(url_for_key("unknown/1/2"), None);
}
#[test]
fn builds_transform_matrix_for_attachment_and_avatar() {
let attachment = cases_for_key(
"attachments/a/b/photo.png",
"plain,resize,format,resize+format",
128,
"webp",
);
assert_eq!(attachment.len(), 4);
assert_eq!(attachment[1].query, "width=128&height=128");
assert_eq!(attachment[3].query, "format=webp&width=128&height=128");
let avatar = cases_for_key("avatars/42/hash", "resize,resize+format", 64, "webp");
assert_eq!(avatar[0].query, "size=64");
assert_eq!(avatar[1].query, "size=64&format=webp");
}
#[test]
fn signs_external_urls_with_urlsafe_components() {
let signed = sign_external_url(
"benchmark-secret",
"http://127.0.0.1:19110/",
"https://example.test/a b.jpg",
)
.unwrap();
let parts = signed.split('/').collect::<Vec<_>>();
assert!(signed.starts_with("http://127.0.0.1:19110/external/"));
assert_eq!(parts[5], "v2");
assert_eq!(
URL_SAFE_NO_PAD.decode(parts[6]).unwrap(),
b"https://example.test/a b.jpg"
);
assert_eq!(URL_SAFE_NO_PAD.decode(parts[4]).unwrap().len(), 32);
}
#[test]
fn summarizes_results_by_prefix() {
let results = vec![
result_for("/attachments/a/b/c.png", vec![]),
result_for("/attachments/a/b/d.png", vec!["bad".to_owned()]),
result_for("/avatars/1/a.webp", vec![]),
];
let summary = summarize_results(&results, 1);
assert_eq!(summary.total, 3);
assert_eq!(summary.issues, 1);
assert_eq!(
summary.by_prefix.get("attachments"),
Some(&PrefixSummary { ok: 1, fail: 1 })
);
assert_eq!(summary.issues_sample.len(), 1);
}
#[test]
fn pixel_similarity_detects_equal_and_different_images() {
let image_a = RgbaImage::from_pixel(2, 2, image::Rgba([0, 0, 0, 255]));
let image_b = RgbaImage::from_pixel(2, 2, image::Rgba([0, 0, 0, 255]));
let image_c = RgbaImage::from_pixel(2, 2, image::Rgba([255, 255, 255, 255]));
assert_eq!(pixel_similarity(&image_a, &image_b), 0.0);
assert!(pixel_similarity(&image_a, &image_c) > 0.70);
}
fn result_for(path: &str, issues: Vec<String>) -> StressResult {
StressResult {
case: StressCase {
label: path.to_owned(),
url_path: path.to_owned(),
query: String::new(),
expect_image: true,
},
v1_status: 200,
v2_status: 200,
v1_ct: "image/png".to_owned(),
v2_ct: "image/png".to_owned(),
v1_size: 10,
v2_size: 10,
issues,
elapsed_v1_ms: 1.0,
elapsed_v2_ms: 1.0,
}
}
}
File diff suppressed because it is too large Load Diff
+115
View File
@@ -0,0 +1,115 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::{Context, Result, bail};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::LazyLock;
pub static ROOT: LazyLock<PathBuf> = LazyLock::new(|| {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.and_then(Path::parent)
.expect("tools/dev has a repository grandparent")
.to_path_buf()
});
pub static DEV_STATE_DIR: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join(".fluxer/dev"));
pub static DEV_CASSANDRA_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("cassandra"));
pub static DEV_GATEWAY_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("gateway"));
pub static DEV_LOG_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("logs"));
pub static DEV_SEAWEEDFS_DIR: LazyLock<PathBuf> = LazyLock::new(|| DEV_STATE_DIR.join("seaweedfs"));
pub static DEV_SEAWEEDFS_PID_FILE: LazyLock<PathBuf> =
LazyLock::new(|| DEV_SEAWEEDFS_DIR.join("seaweedfs.pid"));
pub static DEV_ENV_FILE: LazyLock<PathBuf> =
LazyLock::new(|| ROOT.join("config/env/development.env"));
pub static DEV_LOCAL_ENV_FILE: LazyLock<PathBuf> =
LazyLock::new(|| ROOT.join("config/env/local.env"));
pub static ROOT_LOCAL_ENV_FILE: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join(".env.local"));
pub static TARGET_DIR: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join("target"));
pub static DESKTOP_DIR: LazyLock<PathBuf> = LazyLock::new(|| ROOT.join("fluxer_desktop"));
pub static GATEWAY_CONFIG_DIR: LazyLock<PathBuf> =
LazyLock::new(|| ROOT.join("fluxer_gateway/config"));
pub fn ensure_state_dirs() -> Result<()> {
for path in [
DEV_STATE_DIR.as_path(),
DEV_CASSANDRA_DIR.as_path(),
DEV_GATEWAY_DIR.as_path(),
DEV_LOG_DIR.as_path(),
DEV_SEAWEEDFS_DIR.as_path(),
] {
std::fs::create_dir_all(path)
.with_context(|| format!("failed to create {}", path.display()))?;
}
Ok(())
}
pub fn ensure_writable_dev_paths() -> Result<()> {
for path in [DEV_STATE_DIR.as_path(), TARGET_DIR.as_path()] {
std::fs::create_dir_all(path)
.with_context(|| format!("failed to create {}", path.display()))?;
if is_writable(path) {
continue;
}
let sudo = which("sudo").ok_or_else(|| {
anyhow::anyhow!("{} is not writable and sudo is unavailable", path.display())
})?;
let owner = format!("{}:{}", current_uid(), current_gid());
let status = Command::new(sudo)
.args(["chown", "-R", &owner])
.arg(path)
.status()
.with_context(|| format!("failed to chown {}", path.display()))?;
if !status.success() {
bail!("sudo chown failed for {}", path.display());
}
}
Ok(())
}
pub fn which(name: &str) -> Option<PathBuf> {
let path = std::env::var_os("PATH")?;
for dir in std::env::split_paths(&path) {
let candidate = dir.join(name);
if candidate.is_file() {
return Some(candidate);
}
}
None
}
fn is_writable(path: &Path) -> bool {
let probe = path.join(".fluxer-write-test");
match std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&probe)
{
Ok(_) => {
let _ = std::fs::remove_file(probe);
true
}
Err(_) => false,
}
}
#[cfg(unix)]
fn current_uid() -> u32 {
unsafe { libc::geteuid() }
}
#[cfg(unix)]
fn current_gid() -> u32 {
unsafe { libc::getegid() }
}
#[cfg(not(unix))]
fn current_uid() -> u32 {
0
}
#[cfg(not(unix))]
fn current_gid() -> u32 {
0
}
+321
View File
@@ -0,0 +1,321 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::env::merge_default_env_with_current;
use crate::paths::{DEV_ENV_FILE, DEV_LOCAL_ENV_FILE, ROOT, ROOT_LOCAL_ENV_FILE};
use anyhow::{Context, Result, bail};
use std::collections::{BTreeMap, VecDeque};
use std::net::{TcpStream, ToSocketAddrs};
use std::path::Path;
use std::process::{Command, Output, Stdio};
use std::time::{Duration, Instant};
use tokio::time::sleep;
pub const PNPM_INSTALL_ENV: &[(&str, &str)] = &[
("CI", "true"),
("npm_config_child_concurrency", "2"),
("npm_config_network_concurrency", "8"),
];
pub fn format_command(args: &[impl AsRef<str>]) -> String {
args.iter()
.map(|arg| quote_posix(arg.as_ref()))
.collect::<Vec<_>>()
.join(" ")
}
fn quote_posix(value: &str) -> String {
if value.is_empty() {
return "''".to_owned();
}
if value
.bytes()
.all(|ch| ch.is_ascii_alphanumeric() || b"._+-/:=@%".contains(&ch))
{
return value.to_owned();
}
format!("'{}'", value.replace('\'', "'\"'\"'"))
}
pub fn merged_env(
extra: Option<&[(String, Option<String>)]>,
load_default_env: bool,
) -> Result<BTreeMap<String, String>> {
let mut current: BTreeMap<String, String> = std::env::vars().collect();
if load_default_env {
current = merge_default_env_with_current(
DEV_ENV_FILE.as_path(),
DEV_LOCAL_ENV_FILE.as_path(),
ROOT_LOCAL_ENV_FILE.as_path(),
current,
)?;
}
if let Some(extra) = extra {
for (key, value) in extra {
match value {
Some(value) => {
current.insert(key.clone(), value.clone());
}
None => {
current.remove(key);
}
}
}
}
if load_default_env {
current.insert("FLUXER_SELF_HOSTED".to_owned(), "true".to_owned());
}
Ok(current)
}
pub fn run(args: &[&str]) -> Result<()> {
run_command(args, RunOptions::default()).map(drop)
}
pub fn run_with_env(args: &[&str], env: Vec<(String, Option<String>)>) -> Result<()> {
run_command(
args,
RunOptions {
env,
..RunOptions::default()
},
)
.map(drop)
}
pub fn run_capture(
args: &[&str],
env: Vec<(String, Option<String>)>,
check: bool,
) -> Result<Output> {
run_command(
args,
RunOptions {
env,
check,
capture: true,
..RunOptions::default()
},
)
}
#[derive(Debug)]
pub struct RunOptions<'a> {
pub cwd: &'a Path,
pub env: Vec<(String, Option<String>)>,
pub check: bool,
pub capture: bool,
pub load_default_env: bool,
}
impl Default for RunOptions<'_> {
fn default() -> Self {
Self {
cwd: ROOT.as_path(),
env: Vec::new(),
check: true,
capture: false,
load_default_env: true,
}
}
}
pub fn run_command(args: &[&str], options: RunOptions<'_>) -> Result<Output> {
println!("$ {}", format_command(args));
let env = merged_env(Some(&options.env), options.load_default_env)?;
let mut command = Command::new(args[0]);
command
.args(&args[1..])
.current_dir(options.cwd)
.env_clear()
.envs(env);
if options.capture {
command.stdout(Stdio::piped()).stderr(Stdio::piped());
let output = command
.output()
.with_context(|| format!("failed to run {}", format_command(args)))?;
let mut printed = Vec::new();
printed.extend_from_slice(&output.stdout);
printed.extend_from_slice(&output.stderr);
let text = String::from_utf8_lossy(&printed);
if !text.trim_end().is_empty() {
println!("{}", text.trim_end());
}
if options.check && !output.status.success() {
let code = output.status.code().unwrap_or(-1);
bail!(
"Command failed with exit code {code}: {}",
format_command(args)
);
}
return Ok(output);
}
command
.stdin(Stdio::inherit())
.stdout(Stdio::inherit())
.stderr(Stdio::inherit());
let status = command
.status()
.with_context(|| format!("failed to run {}", format_command(args)))?;
let output = Output {
status,
stdout: Vec::new(),
stderr: Vec::new(),
};
if options.check && !output.status.success() {
let code = output.status.code().unwrap_or(-1);
bail!(
"Command failed with exit code {code}: {}",
format_command(args)
);
}
Ok(output)
}
pub async fn wait_tcp(name: &str, host: &str, port: u16, timeout_secs: u64) -> Result<()> {
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_error = None;
while Instant::now() < deadline {
let address = format!("{host}:{port}");
match address
.to_socket_addrs()
.ok()
.and_then(|mut addrs| addrs.next())
.map(|addr| TcpStream::connect_timeout(&addr, Duration::from_secs(2)))
{
Some(Ok(_)) => {
println!("{name} is reachable at {host}:{port}");
return Ok(());
}
Some(Err(error)) => last_error = Some(error.to_string()),
None => last_error = Some(format!("failed to resolve {address}")),
}
sleep(Duration::from_secs(2)).await;
}
bail!(
"Timed out waiting for {name} at {host}:{port}: {}",
last_error.unwrap_or_else(|| "unknown error".to_owned())
);
}
pub async fn wait_http(name: &str, url: &str, timeout_secs: u64) -> Result<()> {
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()?;
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_error = None;
while Instant::now() < deadline {
match client.get(url).send().await {
Ok(response) if response.status().as_u16() < 500 => {
println!("{name} is reachable at {url}");
return Ok(());
}
Ok(response) => last_error = Some(format!("HTTP {}", response.status())),
Err(error) => last_error = Some(error.to_string()),
}
sleep(Duration::from_secs(2)).await;
}
bail!(
"Timed out waiting for {name} at {url}: {}",
last_error.unwrap_or_else(|| "unknown error".to_owned())
);
}
pub const RESTART_WINDOW: Duration = Duration::from_secs(60);
pub const RESTART_LIMIT: usize = 5;
pub fn restart_budget_exceeded(restarts: &mut VecDeque<Instant>, now: Instant) -> bool {
while restarts
.front()
.is_some_and(|at| now.duration_since(*at) > RESTART_WINDOW)
{
restarts.pop_front();
}
if restarts.len() >= RESTART_LIMIT {
return true;
}
restarts.push_back(now);
false
}
#[cfg(unix)]
pub struct ShutdownSignal {
interrupt: tokio::signal::unix::Signal,
terminate: tokio::signal::unix::Signal,
}
#[cfg(unix)]
impl ShutdownSignal {
pub fn new() -> Result<Self> {
use tokio::signal::unix::{SignalKind, signal};
Ok(Self {
interrupt: signal(SignalKind::interrupt())?,
terminate: signal(SignalKind::terminate())?,
})
}
pub async fn recv(&mut self) -> &'static str {
tokio::select! {
_ = self.interrupt.recv() => "SIGINT",
_ = self.terminate.recv() => "SIGTERM",
}
}
}
#[cfg(not(unix))]
pub struct ShutdownSignal;
#[cfg(not(unix))]
impl ShutdownSignal {
pub fn new() -> Result<Self> {
Ok(Self)
}
pub async fn recv(&mut self) -> &'static str {
let _ = tokio::signal::ctrl_c().await;
"signal"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn formats_posix_commands_for_logs() {
assert_eq!(format_command(&["pnpm", "build"]), "pnpm build");
assert_eq!(
format_command(&["", "two words", "a'b"]),
"'' 'two words' 'a'\"'\"'b'"
);
}
#[test]
fn restart_budget_allows_limited_restarts_within_window() {
let mut restarts = VecDeque::new();
let base = Instant::now();
for offset in 0..RESTART_LIMIT {
assert!(!restart_budget_exceeded(
&mut restarts,
base + Duration::from_secs(offset as u64)
));
}
assert!(restart_budget_exceeded(
&mut restarts,
base + Duration::from_secs(RESTART_LIMIT as u64)
));
}
#[test]
fn restart_budget_resets_after_window_elapses() {
let mut restarts = VecDeque::new();
let base = Instant::now();
for _ in 0..RESTART_LIMIT {
assert!(!restart_budget_exceeded(&mut restarts, base));
}
assert!(!restart_budget_exceeded(
&mut restarts,
base + RESTART_WINDOW + Duration::from_secs(1)
));
}
}
+732
View File
@@ -0,0 +1,732 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::manifest::{ANY_HOST, DEV_PROXY_PORT, LOCAL_APP_URL, PROXY_ROUTES, ProxyRoute};
use anyhow::{Context, Result, bail};
use axum::{
Router,
body::Body,
extract::{ConnectInfo, State},
http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, header},
response::IntoResponse,
routing::any,
};
use hyper_util::rt::TokioIo;
use std::collections::HashMap;
use std::env;
use std::net::SocketAddr;
use std::sync::{LazyLock, Mutex};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use url::Url;
static ROUTE_CURSORS: LazyLock<Mutex<HashMap<&'static str, usize>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[derive(Clone)]
struct ProxyState {
http_client: reqwest::Client,
}
const BLOCKED_REQUEST_HEADERS: &[&str] = &[
"connection",
"content-length",
"host",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"trailers",
"transfer-encoding",
"upgrade",
];
const BLOCKED_RESPONSE_HEADERS: &[&str] = &[
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"trailers",
"transfer-encoding",
"upgrade",
];
const X_FORWARDED_FOR_HEADER: &str = "x-forwarded-for";
pub async fn run_proxy(host: &str, port: u16) -> Result<()> {
let bind = format!("{host}:{port}");
let listener = TcpListener::bind(&bind).await?;
println!("Fluxer dev proxy listening on {}", listener.local_addr()?);
let http_client = reqwest::Client::builder()
.pool_max_idle_per_host(64)
.pool_idle_timeout(Duration::from_secs(90))
.redirect(reqwest::redirect::Policy::none())
.tcp_nodelay(true)
.build()
.context("failed to build dev proxy HTTP client")?;
let app = Router::new()
.fallback(any(proxy_request))
.with_state(ProxyState { http_client });
axum::serve(
listener,
app.into_make_service_with_connect_info::<SocketAddr>(),
)
.await
.context("dev proxy server exited unexpectedly")
}
async fn proxy_request(
State(state): State<ProxyState>,
ConnectInfo(client_addr): ConnectInfo<SocketAddr>,
request: Request<Body>,
) -> Response<Body> {
let request_head = request_head_from_request(&request);
if let Some(location) = tunnel_public_redirect_location(&request_head) {
return redirect_response(&request_head, &location).into_response();
}
let route = route_for_path(&request_head.path);
if is_upgrade_request(&request_head) {
return proxy_upgrade(request, request_head, route, client_addr).await;
}
proxy_http(state, request, request_head, route, client_addr).await
}
async fn proxy_http(
state: ProxyState,
request: Request<Body>,
request_head: RequestHead,
route: &'static ProxyRoute,
client_addr: SocketAddr,
) -> Response<Body> {
let (target_host, target_port) = target_for_route(route);
let target_url = upstream_http_url(&request_head.path, route, target_host, target_port);
let (parts, body) = request.into_parts();
let method = parts.method.clone();
let mut request_builder = state.http_client.request(method.clone(), &target_url);
if method != Method::GET && method != Method::HEAD {
request_builder = request_builder.body(reqwest::Body::wrap_stream(body.into_data_stream()));
}
let mut saw_forwarded_for = false;
for (name, value) in &parts.headers {
let name_str = name.as_str();
if BLOCKED_REQUEST_HEADERS.contains(&name_str) {
continue;
}
if name.as_str().eq_ignore_ascii_case(X_FORWARDED_FOR_HEADER) {
saw_forwarded_for = true;
if let Some(value) = append_forwarded_for(value, client_addr) {
request_builder = request_builder.header(name.clone(), value);
}
} else {
request_builder = request_builder.header(name.clone(), value.clone());
}
}
if !saw_forwarded_for {
request_builder =
request_builder.header(X_FORWARDED_FOR_HEADER, client_addr.ip().to_string());
}
let upstream_response = match request_builder.send().await {
Ok(response) => response,
Err(error) => {
eprintln!("dev proxy HTTP request failed for {target_url}: {error:#}");
return bad_gateway_response(error);
}
};
let status = upstream_response.status();
let headers = upstream_response.headers().clone();
let mut response = Response::new(Body::from_stream(upstream_response.bytes_stream()));
*response.status_mut() = status;
copy_response_headers(&headers, response.headers_mut());
response
}
async fn proxy_upgrade(
mut request: Request<Body>,
request_head: RequestHead,
route: &'static ProxyRoute,
client_addr: SocketAddr,
) -> Response<Body> {
let on_upgrade = hyper::upgrade::on(&mut request);
let (target_host, target_port) = target_for_route(route);
let mut target = match TcpStream::connect((target_host, target_port)).await {
Ok(target) => target,
Err(error) => return bad_gateway_response(error),
};
if let Err(error) = target
.write_all(&rewrite_head(
&request_head,
route,
Some(target_host),
Some(target_port),
Some(&client_addr.ip().to_string()),
))
.await
{
return bad_gateway_response(error);
}
let (head, body_prefix) = match read_http_head(&mut target).await {
Ok(value) => value,
Err(error) => return bad_gateway_response(error),
};
let upstream_response = match parse_response_head(&head) {
Ok(value) => value,
Err(error) => return bad_gateway_response(error),
};
let mut response = Response::new(Body::empty());
*response.status_mut() = upstream_response.status;
copy_upgrade_response_headers(&upstream_response.headers, response.headers_mut());
if upstream_response.status != StatusCode::SWITCHING_PROTOCOLS {
*response.body_mut() = Body::from(body_prefix);
return response;
}
tokio::spawn(async move {
let result: Result<()> = async move {
let upgraded = on_upgrade.await?;
let mut client = TokioIo::new(upgraded);
if !body_prefix.is_empty() {
client.write_all(&body_prefix).await?;
}
tokio::io::copy_bidirectional(&mut client, &mut target).await?;
Ok(())
}
.await;
if let Err(error) = result {
eprintln!("dev proxy upgrade bridge failed: {error:#}");
}
});
response
}
fn upstream_http_url(
request_path: &str,
route: &ProxyRoute,
target_host: &str,
target_port: u16,
) -> String {
format!(
"http://{target_host}:{target_port}{}",
rewrite_path(request_path, route)
)
}
fn request_head_from_request(request: &Request<Body>) -> RequestHead {
RequestHead {
method: request.method().as_str().to_owned(),
path: request
.uri()
.path_and_query()
.map(|value| value.as_str().to_owned())
.unwrap_or_else(|| request.uri().to_string()),
version: format!("{:?}", request.version()),
headers: request
.headers()
.iter()
.map(|(name, value)| {
(
name.as_str().to_owned(),
value.to_str().unwrap_or_default().to_owned(),
)
})
.collect(),
}
}
fn append_forwarded_for(value: &HeaderValue, client_addr: SocketAddr) -> Option<HeaderValue> {
let value = value.to_str().ok()?;
HeaderValue::from_str(&format!("{value}, {}", client_addr.ip())).ok()
}
fn copy_response_headers(source: &HeaderMap, target: &mut HeaderMap) {
for (name, value) in source {
if BLOCKED_RESPONSE_HEADERS.contains(&name.as_str()) {
continue;
}
target.append(name.clone(), value.clone());
}
}
fn copy_upgrade_response_headers(source: &[(String, String)], target: &mut HeaderMap) {
for (name, value) in source {
let Ok(name) = axum::http::HeaderName::from_bytes(name.as_bytes()) else {
continue;
};
let Ok(value) = HeaderValue::from_str(value) else {
continue;
};
target.append(name, value);
}
}
fn bad_gateway_response(error: impl std::fmt::Display) -> Response<Body> {
(
StatusCode::BAD_GATEWAY,
[(
header::CONTENT_TYPE,
HeaderValue::from_static("text/plain; charset=utf-8"),
)],
format!("{error}\n"),
)
.into_response()
}
async fn read_http_head(stream: &mut TcpStream) -> Result<(Vec<u8>, Vec<u8>)> {
let mut head = Vec::new();
let mut buf = [0_u8; 4096];
loop {
let count = stream.read(&mut buf).await?;
if count == 0 {
bail!("connection closed before response head");
}
head.extend_from_slice(&buf[..count]);
if head.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
if head.len() > 64 * 1024 {
bail!("response head exceeded 64KiB");
}
}
let header_end = head
.windows(4)
.position(|window| window == b"\r\n\r\n")
.map(|index| index + 4)
.context("missing response header terminator")?;
let body_prefix = head.split_off(header_end);
Ok((head, body_prefix))
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ResponseHead {
status: StatusCode,
headers: Vec<(String, String)>,
}
fn parse_response_head(head: &[u8]) -> Result<ResponseHead> {
let text = String::from_utf8_lossy(head);
let mut lines = text.split("\r\n");
let status_line = lines.next().unwrap_or_default();
let parts = status_line.splitn(3, ' ').collect::<Vec<_>>();
if parts.len() < 2 {
bail!("invalid response status line: {status_line:?}");
}
let status_code = parts[1]
.parse::<u16>()
.with_context(|| format!("invalid response status code: {:?}", parts[1]))?;
let status = StatusCode::from_u16(status_code)
.with_context(|| format!("unsupported response status code: {status_code}"))?;
let headers = lines
.filter_map(|line| {
if line.is_empty() {
return None;
}
let (name, value) = line.split_once(':')?;
(!name.is_empty()).then(|| (name.to_owned(), value.trim().to_owned()))
})
.collect();
Ok(ResponseHead { status, headers })
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RequestHead {
pub method: String,
pub path: String,
pub version: String,
pub headers: Vec<(String, String)>,
}
pub fn parse_request_head(head: &[u8]) -> Result<RequestHead> {
let text = String::from_utf8_lossy(head);
let mut lines = text.split("\r\n");
let request_line = lines.next().unwrap_or_default();
let parts = request_line.splitn(3, ' ').collect::<Vec<_>>();
if parts.len() != 3 {
bail!("invalid request line: {request_line:?}");
}
let headers = lines
.filter_map(|line| {
if line.is_empty() {
return None;
}
let (name, value) = line.split_once(':')?;
(!name.is_empty()).then(|| (name.to_owned(), value.trim().to_owned()))
})
.collect();
Ok(RequestHead {
method: parts[0].to_owned(),
path: parts[1].to_owned(),
version: parts[2].to_owned(),
headers,
})
}
pub fn tunnel_public_redirect_location(request: &RequestHead) -> Option<String> {
let public_url = env::var("FLUXER_PUBLIC_URL").ok()?;
redirect_location_for_public_url(request, &public_url)
}
pub fn redirect_location_for_public_url(request: &RequestHead, public_url: &str) -> Option<String> {
if !request.method.eq_ignore_ascii_case("GET") && !request.method.eq_ignore_ascii_case("HEAD") {
return None;
}
if !is_local_dev_host(request) || !accepts_html(request) {
return None;
}
let parsed_public_url = Url::parse(public_url).ok()?;
if is_local_url(&parsed_public_url) {
return None;
}
let path = normalize_request_target(&request.path);
Some(format!(
"{}{}",
parsed_public_url.as_str().trim_end_matches('/'),
path
))
}
fn redirect_response(request: &RequestHead, location: &str) -> Response<Body> {
let body = if request.method.eq_ignore_ascii_case("HEAD") {
String::new()
} else {
format!("Redirecting to {location}\n")
};
let mut response = Response::new(Body::from(body));
*response.status_mut() = StatusCode::FOUND;
if let Ok(value) = HeaderValue::from_str(location) {
response.headers_mut().insert(header::LOCATION, value);
}
response
.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("text/plain; charset=utf-8"),
);
response
}
fn is_local_dev_host(request: &RequestHead) -> bool {
let Some(host) = header_value(request, "host") else {
return false;
};
matches!(
host_without_port(host).as_deref(),
Some("localhost" | "127.0.0.1" | "::1" | "0.0.0.0")
)
}
fn is_local_url(url: &Url) -> bool {
if url.as_str().trim_end_matches('/') == LOCAL_APP_URL {
return true;
}
matches!(
url.host_str(),
Some("localhost" | "127.0.0.1" | "::1" | "0.0.0.0")
)
}
fn accepts_html(request: &RequestHead) -> bool {
header_value(request, "accept")
.map(|value| value.to_ascii_lowercase().contains("text/html"))
.unwrap_or(false)
}
fn is_upgrade_request(request: &RequestHead) -> bool {
let connection_upgrade = header_value(request, "connection")
.map(|value| value.to_ascii_lowercase().contains("upgrade"))
.unwrap_or(false);
connection_upgrade && header_value(request, "upgrade").is_some()
}
fn header_value<'a>(request: &'a RequestHead, name: &str) -> Option<&'a str> {
request
.headers
.iter()
.find(|(header_name, _)| header_name.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
}
fn host_without_port(host: &str) -> Option<String> {
let host = host.trim().to_ascii_lowercase();
if host.is_empty() {
return None;
}
if let Some(rest) = host.strip_prefix('[') {
return rest.split_once(']').map(|(host, _)| host.to_owned());
}
Some(host.split(':').next().unwrap_or(&host).to_owned())
}
pub fn route_for_path(path: &str) -> &'static ProxyRoute {
let normalized_path = normalize_request_target(path);
let parsed_path = Url::parse(&format!("http://fluxer.local{normalized_path}"))
.map(|url| url.path().to_owned())
.unwrap_or_else(|_| {
normalized_path
.split('?')
.next()
.unwrap_or(&normalized_path)
.to_owned()
});
for route in PROXY_ROUTES {
if route.prefix == "/" {
return route;
}
if parsed_path == route.prefix
|| parsed_path.starts_with(&format!("{}/", route.prefix))
|| (route.prefix.ends_with('-') && parsed_path.starts_with(route.prefix))
{
return route;
}
}
PROXY_ROUTES.last().expect("proxy has fallback route")
}
pub fn target_for_route(route: &'static ProxyRoute) -> (&'static str, u16) {
if route.alternate_ports.is_empty() {
return (route.host, route.port);
}
let ports = std::iter::once(route.port)
.chain(route.alternate_ports.iter().copied())
.collect::<Vec<_>>();
let mut cursors = ROUTE_CURSORS.lock().expect("route cursor lock poisoned");
let cursor = *cursors.get(route.prefix).unwrap_or(&0);
cursors.insert(route.prefix, (cursor + 1) % ports.len());
(route.host, ports[cursor])
}
pub fn rewrite_path(path: &str, route: &ProxyRoute) -> String {
let normalized_path = normalize_request_target(path);
if !route.strip_prefix {
return normalized_path;
}
let (parsed_path, query) = split_path_query(&normalized_path);
let next_path = parsed_path
.strip_prefix(route.prefix)
.unwrap_or(parsed_path);
let next_path = if next_path.is_empty() { "/" } else { next_path };
match query {
Some(query) => format!("{next_path}?{query}"),
None => next_path.to_owned(),
}
}
fn normalize_request_target(request_target: &str) -> String {
let Ok(url) = Url::parse(request_target) else {
return request_target.to_owned();
};
if url.scheme() != "http" && url.scheme() != "https" {
return request_target.to_owned();
}
let mut path = url.path().to_owned();
if path.is_empty() {
path.push('/');
}
if let Some(query) = url.query() {
path.push('?');
path.push_str(query);
}
path
}
fn split_path_query(path: &str) -> (&str, Option<&str>) {
path.split_once('?')
.map(|(path, query)| (path, Some(query)))
.unwrap_or((path, None))
}
pub fn rewrite_head(
request: &RequestHead,
route: &ProxyRoute,
target_host: Option<&str>,
target_port: Option<u16>,
client_ip: Option<&str>,
) -> Vec<u8> {
let host = target_host.unwrap_or(route.host);
let port = target_port.unwrap_or(route.port);
let is_upgrade = request
.headers
.iter()
.any(|(name, _)| name.eq_ignore_ascii_case("upgrade"));
let mut lines = vec![format!(
"{} {} {}",
request.method,
rewrite_path(&request.path, route),
request.version
)];
let mut saw_forwarded_for = false;
let mut saw_host = false;
let mut saw_connection = false;
for (name, value) in &request.headers {
if name.eq_ignore_ascii_case("host") {
saw_host = true;
lines.push(format!("Host: {host}:{port}"));
} else if name.eq_ignore_ascii_case("connection") {
saw_connection = true;
lines.push(format!(
"Connection: {}",
if is_upgrade { "Upgrade" } else { "close" }
));
} else if name.eq_ignore_ascii_case("x-forwarded-for") {
saw_forwarded_for = true;
if let Some(client_ip) = client_ip {
lines.push(format!("{name}: {value}, {client_ip}"));
} else {
lines.push(format!("{name}: {value}"));
}
} else {
lines.push(format!("{name}: {value}"));
}
}
if !saw_host {
lines.push(format!("Host: {host}:{port}"));
}
if !saw_connection {
lines.push(format!(
"Connection: {}",
if is_upgrade { "Upgrade" } else { "close" }
));
}
if let Some(client_ip) = client_ip
&& !saw_forwarded_for
{
lines.push(format!("X-Forwarded-For: {client_ip}"));
}
lines.push(String::new());
lines.push(String::new());
lines.join("\r\n").into_bytes()
}
#[allow(dead_code)]
pub fn default_bind() -> (&'static str, u16) {
(ANY_HOST, DEV_PROXY_PORT)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn routes_and_rewrites_prefixes() {
let api = route_for_path("/api/users?x=1");
assert_eq!(api.prefix, "/api");
assert_eq!(rewrite_path("/api/users?x=1", api), "/users?x=1");
let asset = route_for_path("/assets/app.js");
assert_eq!(rewrite_path("/assets/app.js", asset), "/assets/app.js");
let lazy = route_for_path("/lazy-compilation-using-foo");
assert_eq!(lazy.prefix, "/lazy-compilation-using-");
let devmail = route_for_path("/devmail/messages");
assert_eq!(devmail.prefix, "/devmail");
assert_eq!(
rewrite_path("/devmail/messages?x=1", devmail),
"/devmail/messages?x=1"
);
let admin = route_for_path("/admin/users");
assert_eq!(admin.prefix, "/admin");
assert_eq!(rewrite_path("/admin/users", admin), "/users");
let absolute_gateway = route_for_path("https://dev.example.test/gateway?v=1&encoding=json");
assert_eq!(absolute_gateway.prefix, "/gateway");
assert_eq!(
rewrite_path(
"https://dev.example.test/gateway?v=1&encoding=json",
absolute_gateway
),
"/?v=1&encoding=json"
);
}
#[test]
fn rewrites_host_connection_and_forwarded_for() {
let request = RequestHead {
method: "GET".to_owned(),
path: "/api/_health".to_owned(),
version: "HTTP/1.1".to_owned(),
headers: vec![
("Host".to_owned(), "localhost:8088".to_owned()),
("Connection".to_owned(), "keep-alive".to_owned()),
("X-Forwarded-For".to_owned(), "1.1.1.1".to_owned()),
],
};
let head = String::from_utf8(rewrite_head(
&request,
route_for_path("/api/_health"),
Some("api"),
Some(8080),
Some("2.2.2.2"),
))
.unwrap();
assert!(head.contains("GET /_health HTTP/1.1\r\n"));
assert!(head.contains("Host: api:8080\r\n"));
assert!(head.contains("Connection: close\r\n"));
assert!(head.contains("X-Forwarded-For: 1.1.1.1, 2.2.2.2\r\n"));
}
#[test]
fn redirects_local_html_navigation_to_public_tunnel_url() {
let request = RequestHead {
method: "GET".to_owned(),
path: "/channels/@me?x=1".to_owned(),
version: "HTTP/1.1".to_owned(),
headers: vec![
("Host".to_owned(), "localhost:8088".to_owned()),
(
"Accept".to_owned(),
"text/html,application/xhtml+xml".to_owned(),
),
],
};
assert_eq!(
redirect_location_for_public_url(&request, "https://dev.example.test"),
Some("https://dev.example.test/channels/@me?x=1".to_owned())
);
}
#[test]
fn public_tunnel_redirect_ignores_assets_and_local_public_url() {
let asset = RequestHead {
method: "GET".to_owned(),
path: "/avatars/4.png".to_owned(),
version: "HTTP/1.1".to_owned(),
headers: vec![
("Host".to_owned(), "localhost:8088".to_owned()),
(
"Accept".to_owned(),
"image/avif,image/webp,image/apng,image/*,*/*;q=0.8".to_owned(),
),
],
};
assert_eq!(
redirect_location_for_public_url(&asset, "https://dev.example.test"),
None
);
let local_public_url = RequestHead {
method: "GET".to_owned(),
path: "/channels/@me".to_owned(),
version: "HTTP/1.1".to_owned(),
headers: vec![
("Host".to_owned(), "localhost:8088".to_owned()),
("Accept".to_owned(), "text/html".to_owned()),
],
};
assert_eq!(
redirect_location_for_public_url(&local_public_url, LOCAL_APP_URL),
None
);
}
}
+732
View File
@@ -0,0 +1,732 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::manifest::{DEV_PROXY_PORT, MEDIA_PROXY_PORT, RustServiceSpec, rust_services};
use crate::paths::ROOT;
use crate::proc::{
RESTART_LIMIT, RESTART_WINDOW, RunOptions, ShutdownSignal, format_command, merged_env,
restart_budget_exceeded, run_command,
};
use anyhow::{Result, bail};
use std::collections::{BTreeSet, VecDeque};
use std::env;
#[cfg(target_os = "linux")]
use std::fs;
use std::net::{SocketAddr, TcpListener};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
use tokio::time::sleep;
const SERVICE_RESTART_PORT_WAIT: Duration = Duration::from_secs(75);
struct SupervisedService {
spec: RustServiceSpec,
mode: &'static str,
port: u16,
child: Child,
restarts: VecDeque<Instant>,
}
impl SupervisedService {
fn label(&self) -> String {
format!("{}:{}", self.spec.name, self.mode)
}
}
pub async fn run_rust_services(service_names: &[String]) -> Result<i32> {
let selected = select_services(service_names)?;
cleanup_orphaned_service_processes(&selected).await?;
wait_for_service_ports_available(&selected)?;
build_services(&selected)?;
let mut shutdown = ShutdownSignal::new()?;
let mut supervised = Vec::new();
for spec in &selected {
for (mode, port) in [("router", spec.port_base), ("shard", spec.port_base + 1)] {
supervised.push(SupervisedService {
spec: spec.clone(),
mode,
port,
child: start_service(spec, mode, port)?,
restarts: VecDeque::new(),
});
}
}
loop {
if let Err(error) = restart_exited_services(&mut supervised).await {
stop_supervised_services(&mut supervised);
return Err(error);
}
tokio::select! {
signal = shutdown.recv() => {
println!("Received {signal}; stopping Rust service tasks...");
stop_supervised_services(&mut supervised);
return Ok(0);
}
_ = sleep(Duration::from_millis(500)) => {}
}
}
}
async fn restart_exited_services(supervised: &mut [SupervisedService]) -> Result<()> {
for entry in supervised.iter_mut() {
let Some(status) = entry.child.try_wait()? else {
continue;
};
if restart_budget_exceeded(&mut entry.restarts, Instant::now()) {
bail!(
"Rust service {} exited with {status} after {RESTART_LIMIT} restarts within {}s; giving up",
entry.label(),
RESTART_WINDOW.as_secs()
);
}
println!(
"[{}] exited with {status}; restarting service",
entry.label()
);
wait_for_port_available(&entry.label(), entry.port).await?;
entry.child = start_service(&entry.spec, entry.mode, entry.port)?;
}
Ok(())
}
async fn wait_for_port_available(label: &str, port: u16) -> Result<()> {
let deadline = Instant::now() + SERVICE_RESTART_PORT_WAIT;
loop {
if can_bind_tcp_port(port) {
return Ok(());
}
if Instant::now() >= deadline {
bail!("port {port} for Rust service {label} did not become available");
}
sleep(Duration::from_millis(500)).await;
}
}
fn stop_supervised_services(supervised: &mut [SupervisedService]) {
let mut children = supervised
.iter_mut()
.map(|entry| &mut entry.child)
.collect::<Vec<_>>();
let mut processes = Vec::new();
for child in children.drain(..) {
processes.push(child);
}
crate::gateway::stop_child_processes(&mut processes);
}
pub fn select_services(service_names: &[String]) -> Result<Vec<RustServiceSpec>> {
let services = rust_services();
if service_names.is_empty() {
return Ok(services);
}
let unknown = service_names
.iter()
.filter(|name| !services.iter().any(|spec| spec.name == name.as_str()))
.cloned()
.collect::<Vec<_>>();
if !unknown.is_empty() {
let available = services
.iter()
.map(|spec| spec.name)
.collect::<Vec<_>>()
.join(", ");
bail!(
"Unknown Rust service(s): {}. Available: {available}",
unknown.join(", ")
);
}
Ok(service_names
.iter()
.filter_map(|name| {
services
.iter()
.find(|spec| spec.name == name.as_str())
.cloned()
})
.collect())
}
fn build_services(services: &[RustServiceSpec]) -> Result<()> {
let mut packages = Vec::new();
let mut seen = BTreeSet::new();
for spec in services {
if seen.insert(spec.package) {
packages.extend(["-p".to_owned(), spec.package.to_owned()]);
}
}
let mut args = vec!["cargo".to_owned(), "build".to_owned()];
args.extend(packages);
let refs = args.iter().map(String::as_str).collect::<Vec<_>>();
run_command(&refs, RunOptions::default()).map(drop)
}
fn start_service(spec: &RustServiceSpec, mode: &str, port: u16) -> Result<Child> {
let args = service_command(spec);
let env = merged_env(Some(&service_env(spec, mode, port)), true)?;
let label = format!("{}:{mode}", spec.name);
println!("[{label}] $ {}", format_command(&args));
let mut command = Command::new(&args[0]);
command
.args(&args[1..])
.current_dir(ROOT.as_path())
.env_clear()
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
unsafe {
use std::os::unix::process::CommandExt;
command.pre_exec(|| {
libc::setsid();
Ok(())
});
}
let mut child = command.spawn()?;
if let Some(stdout) = child.stdout.take() {
let stdout_label = label.clone();
std::thread::spawn(move || prefix_output(&stdout_label, stdout));
}
if let Some(stderr) = child.stderr.take() {
std::thread::spawn(move || prefix_output(&label, stderr));
}
Ok(child)
}
pub fn service_command(spec: &RustServiceSpec) -> Vec<String> {
if env::var("FLUXER_DEV_RUST_HOT_RELOAD")
.map(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true" | "yes"))
.unwrap_or(false)
{
let rel = spec.path.strip_prefix(ROOT.as_path()).unwrap_or(&spec.path);
return vec![
"cargo".to_owned(),
"watch".to_owned(),
"-w".to_owned(),
rel.join("src").display().to_string(),
"-w".to_owned(),
rel.join("Cargo.toml").display().to_string(),
"-w".to_owned(),
"fluxer_svc/src".to_owned(),
"-w".to_owned(),
"fluxer_svc/Cargo.toml".to_owned(),
"-x".to_owned(),
format!("run -p {}", spec.package),
];
}
vec![
"cargo".to_owned(),
"run".to_owned(),
"-p".to_owned(),
spec.package.to_owned(),
]
}
pub fn service_env(spec: &RustServiceSpec, mode: &str, port: u16) -> Vec<(String, Option<String>)> {
let mut envs = vec![
("FLUXER_SVC_NAME".to_owned(), Some(spec.name.to_owned())),
("FLUXER_SVC_MODE".to_owned(), Some(mode.to_owned())),
("FLUXER_SVC_PORT".to_owned(), Some(port.to_string())),
(
"FLUXER_SVC_LISTEN_HOST".to_owned(),
Some("0.0.0.0".to_owned()),
),
("FLUXER_SVC_SHARD_COUNT".to_owned(), Some("1".to_owned())),
(
"FLUXER_SVC_NATS_URL".to_owned(),
Some(
env::var("FLUXER_SVC_NATS_URL")
.or_else(|_| env::var("FLUXER_NATS_URL"))
.unwrap_or_else(|_| "nats://nats:4222".to_owned()),
),
),
(
"FLUXER_CASSANDRA_HOSTS".to_owned(),
Some(env::var("FLUXER_CASSANDRA_HOSTS").unwrap_or_else(|_| "cassandra".to_owned())),
),
(
"FLUXER_CASSANDRA_KEYSPACE".to_owned(),
Some(env::var("FLUXER_CASSANDRA_KEYSPACE").unwrap_or_else(|_| "fluxer".to_owned())),
),
(
"FLUXER_CASSANDRA_PORT".to_owned(),
Some(env::var("FLUXER_CASSANDRA_PORT").unwrap_or_else(|_| "9042".to_owned())),
),
];
if mode == "shard" {
envs.push(("FLUXER_SVC_SHARD_ID".to_owned(), Some("0".to_owned())));
}
if spec.name == "unfurl" {
envs.extend([
(
"FLUXER_MEDIA_PROXY_ENDPOINT".to_owned(),
Some(
env::var("FLUXER_MEDIA_PROXY_ENDPOINT")
.unwrap_or_else(|_| format!("http://127.0.0.1:{MEDIA_PROXY_PORT}")),
),
),
(
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT".to_owned(),
Some(
env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
.or_else(|_| env::var("FLUXER_MEDIA_ENDPOINT"))
.unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}/media")),
),
),
(
"FLUXER_STATIC_CDN_ENDPOINT".to_owned(),
Some(
env::var("FLUXER_STATIC_CDN_ENDPOINT")
.or_else(|_| env::var("FLUXER_PUBLIC_URL"))
.unwrap_or_else(|_| format!("http://localhost:{DEV_PROXY_PORT}")),
),
),
]);
}
envs
}
fn prefix_output(label: &str, reader: impl std::io::Read) {
use std::io::{BufRead, BufReader};
for line in BufReader::new(reader).lines().map_while(|line| line.ok()) {
println!("[{label}] {line}");
}
}
fn wait_for_service_ports_available(services: &[RustServiceSpec]) -> Result<()> {
let conflicts = services
.iter()
.flat_map(|spec| {
[
(
!can_bind_tcp_port(spec.port_base),
format!("{}:router={}", spec.name, spec.port_base),
),
(
!can_bind_tcp_port(spec.port_base + 1),
format!("{}:shard={}", spec.name, spec.port_base + 1),
),
]
})
.filter_map(|(conflict, label)| conflict.then_some(label))
.collect::<Vec<_>>();
if !conflicts.is_empty() {
bail!(
"Rust service port(s) already in use: {}. Stop the conflicting process before starting rust-services.",
conflicts.join(", ")
);
}
Ok(())
}
fn can_bind_tcp_port(port: u16) -> bool {
TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], port))).is_ok()
}
#[cfg(target_os = "linux")]
async fn cleanup_orphaned_service_processes(services: &[RustServiceSpec]) -> Result<()> {
let leaders = orphaned_service_leaders(services)?;
if leaders.is_empty() {
return Ok(());
}
let pids = leaders.iter().map(|leader| leader.pid).collect::<Vec<_>>();
println!(
"Stopping orphaned Rust service process group(s): {}",
format_pids(&pids)
);
let term_failed = signal_process_groups(&pids, libc::SIGTERM);
if !term_failed.is_empty() {
println!(
"SIGTERM delivery failed for orphaned Rust service pid(s): {}",
format_pids(&term_failed)
);
}
let deadline = Instant::now() + Duration::from_secs(5);
loop {
let remaining = surviving_service_group_pids(&leaders)?;
if remaining.is_empty() {
return Ok(());
}
if Instant::now() >= deadline {
let kill_failed = signal_process_groups(&remaining, libc::SIGKILL);
sleep(Duration::from_millis(200)).await;
let survivors = surviving_service_group_pids(&leaders)?;
if survivors.is_empty() {
return Ok(());
}
let delivery_failed = kill_failed
.into_iter()
.filter(|pid| survivors.contains(pid))
.collect::<Vec<_>>();
if delivery_failed.is_empty() {
bail!(
"orphaned Rust service process(es) survived SIGKILL: {}",
format_pids(&survivors)
);
}
bail!(
"orphaned Rust service process(es) survived SIGKILL: {} (signal delivery failed for: {})",
format_pids(&survivors),
format_pids(&delivery_failed)
);
}
sleep(Duration::from_millis(200)).await;
}
}
#[cfg(not(target_os = "linux"))]
async fn cleanup_orphaned_service_processes(_services: &[RustServiceSpec]) -> Result<()> {
Ok(())
}
#[cfg(target_os = "linux")]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RustServiceLeader {
pid: i32,
starttime: u64,
}
#[cfg(target_os = "linux")]
fn orphaned_service_leaders(services: &[RustServiceSpec]) -> Result<Vec<RustServiceLeader>> {
let binaries = services
.iter()
.map(|spec| format!("target/debug/{}", spec.package))
.collect::<BTreeSet<_>>();
let mut leaders = Vec::new();
for entry in fs::read_dir("/proc")? {
let Ok(entry) = entry else {
continue;
};
let Some(pid) = entry
.file_name()
.to_str()
.and_then(|name| name.parse::<i32>().ok())
else {
continue;
};
if proc_parent_pid(pid) != Some(1) {
continue;
}
if !cmdline_has_service_binary(&proc_cmdline(pid), &binaries) {
continue;
}
if let Some(starttime) = proc_stat_starttime(pid) {
leaders.push(RustServiceLeader { pid, starttime });
}
}
leaders.sort_unstable_by_key(|leader| leader.pid);
Ok(leaders)
}
#[cfg(target_os = "linux")]
fn format_pids(pids: &[i32]) -> String {
if pids.is_empty() {
return "none".to_owned();
}
pids.iter()
.map(i32::to_string)
.collect::<Vec<_>>()
.join(", ")
}
#[cfg(any(target_os = "linux", test))]
fn cmdline_has_service_binary(args: &[String], binaries: &BTreeSet<String>) -> bool {
args.first()
.map(|arg| binaries.iter().any(|binary| arg.ends_with(binary)))
.unwrap_or(false)
}
#[cfg(target_os = "linux")]
fn proc_cmdline(pid: i32) -> Vec<String> {
fs::read(format!("/proc/{pid}/cmdline"))
.unwrap_or_default()
.split(|byte| *byte == 0)
.filter(|arg| !arg.is_empty())
.map(|arg| String::from_utf8_lossy(arg).into_owned())
.collect()
}
#[cfg(target_os = "linux")]
fn proc_parent_pid(pid: i32) -> Option<i32> {
fs::read_to_string(format!("/proc/{pid}/status"))
.ok()?
.lines()
.find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok())
}
#[cfg(target_os = "linux")]
fn process_exists(pid: i32) -> bool {
assert!(pid > 0);
match proc_stat_state_and_pgid(pid) {
Some((state, _pgid)) => !proc_stat_state_is_dead(state),
None => false,
}
}
#[cfg(target_os = "linux")]
fn proc_stat_state_and_pgid(pid: i32) -> Option<(char, i32)> {
assert!(pid > 0);
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
parse_proc_stat_state_and_pgid(&stat)
}
#[cfg(any(target_os = "linux", test))]
fn parse_proc_stat_state_and_pgid(stat: &str) -> Option<(char, i32)> {
let (_, after_comm) = stat.rsplit_once(')')?;
let mut fields = after_comm.split_ascii_whitespace();
let state = fields.next()?.chars().next()?;
let _ppid: i32 = fields.next()?.parse().ok()?;
let pgid: i32 = fields.next()?.parse().ok()?;
Some((state, pgid))
}
#[cfg(target_os = "linux")]
fn proc_stat_starttime(pid: i32) -> Option<u64> {
assert!(pid > 0);
let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
parse_proc_stat_starttime(&stat)
}
#[cfg(any(target_os = "linux", test))]
fn parse_proc_stat_starttime(stat: &str) -> Option<u64> {
let (_, after_comm) = stat.rsplit_once(')')?;
after_comm.split_ascii_whitespace().nth(19)?.parse().ok()
}
#[cfg(any(target_os = "linux", test))]
fn proc_stat_state_is_dead(state: char) -> bool {
state == 'Z' || state == 'X' || state == 'x'
}
#[cfg(target_os = "linux")]
fn surviving_service_group_pids(leaders: &[RustServiceLeader]) -> Result<Vec<i32>> {
let leader_pids = leaders
.iter()
.map(|leader| leader.pid)
.collect::<std::collections::HashSet<_>>();
let mut survivors = Vec::new();
for entry in fs::read_dir("/proc")? {
let Ok(entry) = entry else {
continue;
};
let Some(pid) = entry
.file_name()
.to_str()
.and_then(|name| name.parse::<i32>().ok())
else {
continue;
};
let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else {
continue;
};
let Some((state, pgid)) = parse_proc_stat_state_and_pgid(&stat) else {
continue;
};
if proc_stat_state_is_dead(state) {
continue;
}
if leader_pids.contains(&pgid) {
survivors.push(pid);
continue;
}
let leader = leaders.iter().find(|leader| leader.pid == pid);
if let Some(leader) = leader
&& parse_proc_stat_starttime(&stat) == Some(leader.starttime)
{
survivors.push(pid);
}
}
survivors.sort_unstable();
Ok(survivors)
}
#[cfg(target_os = "linux")]
fn signal_process_groups(pids: &[i32], signal: i32) -> Vec<i32> {
assert!(signal == libc::SIGTERM || signal == libc::SIGKILL);
let mut failed = Vec::with_capacity(pids.len());
for pid in pids {
assert!(*pid > 0);
let group_result = unsafe { libc::kill(-pid, signal) };
if group_result == 0 {
continue;
}
let direct_result = unsafe { libc::kill(*pid, signal) };
if direct_result == 0 {
continue;
}
if process_exists(*pid) {
failed.push(*pid);
}
}
failed
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
static ENV_LOCK: Mutex<()> = Mutex::new(());
fn with_public_env(
overrides: &[(&str, Option<&str>)],
assertions: impl FnOnce(Vec<(String, Option<String>)>),
) {
let _guard = ENV_LOCK.lock().unwrap();
let keys = [
"FLUXER_MEDIA_PROXY_ENDPOINT",
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
"FLUXER_MEDIA_ENDPOINT",
"FLUXER_STATIC_CDN_ENDPOINT",
"FLUXER_PUBLIC_URL",
];
let saved = keys
.iter()
.map(|key| (*key, env::var(key).ok()))
.collect::<Vec<_>>();
for key in keys {
unsafe {
env::remove_var(key);
}
}
for (key, value) in overrides {
if let Some(value) = value {
unsafe {
env::set_var(key, value);
}
}
}
let spec = rust_services()
.into_iter()
.find(|spec| spec.name == "unfurl")
.unwrap();
let env = service_env(&spec, "router", spec.port_base);
for (key, value) in saved {
match value {
Some(value) => unsafe {
env::set_var(key, value);
},
None => unsafe {
env::remove_var(key);
},
}
}
assertions(env);
}
#[test]
fn rejects_unknown_services() {
let err = select_services(&["bogus".to_owned()])
.unwrap_err()
.to_string();
assert!(err.contains("Unknown Rust service"));
}
#[test]
fn builds_unfurl_env_with_media_endpoints() {
with_public_env(&[], |env| {
assert!(
env.iter()
.any(|(key, value)| key == "FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT"
&& value.as_deref() == Some("http://localhost:8088/media"))
);
assert!(
env.iter()
.any(|(key, value)| key == "FLUXER_STATIC_CDN_ENDPOINT"
&& value.as_deref() == Some("http://localhost:8088"))
);
});
}
#[test]
fn unfurl_env_uses_public_url_overrides() {
with_public_env(
&[
(
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
Some("https://dev.example.com/media"),
),
(
"FLUXER_STATIC_CDN_ENDPOINT",
Some("https://dev.example.com"),
),
],
|env| {
assert!(
env.iter()
.any(|(key, value)| key == "FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT"
&& value.as_deref() == Some("https://dev.example.com/media"))
);
assert!(
env.iter()
.any(|(key, value)| key == "FLUXER_STATIC_CDN_ENDPOINT"
&& value.as_deref() == Some("https://dev.example.com"))
);
},
);
}
#[test]
fn cmdline_binary_match_uses_selected_service_binary_suffix() {
let binaries = BTreeSet::from(["target/debug/fluxer_messages".to_owned()]);
assert!(cmdline_has_service_binary(
&["/workspaces/fluxer/target/debug/fluxer_messages".to_owned()],
&binaries
));
assert!(!cmdline_has_service_binary(
&["/workspaces/fluxer/target/debug/fluxer_users".to_owned()],
&binaries
));
}
#[test]
fn proc_stat_parsing_handles_parentheses_and_spaces_in_comm() {
assert_eq!(
parse_proc_stat_state_and_pgid("99 (spaced comm) T 1 99 99 0 -1"),
Some(('T', 99))
);
assert_eq!(
parse_proc_stat_state_and_pgid("99 (weird) comm (name) Z 1 42 42 0 -1"),
Some(('Z', 42))
);
}
#[test]
fn proc_stat_parsing_rejects_malformed_lines() {
assert_eq!(parse_proc_stat_state_and_pgid(""), None);
assert_eq!(parse_proc_stat_state_and_pgid("1234 (svc)"), None);
assert_eq!(parse_proc_stat_state_and_pgid("1234 (svc) S"), None);
assert_eq!(parse_proc_stat_state_and_pgid("1234 (svc) S 1"), None);
assert_eq!(
parse_proc_stat_state_and_pgid("1234 (svc) S 1 not-a-pgid"),
None
);
}
#[test]
fn proc_stat_parsing_extracts_starttime() {
let stat = "1234 (svc) S 1 1234 1234 0 -1 4194560 0 0 0 0 5 3 0 0 20 0 30 0 12345678 4096";
assert_eq!(parse_proc_stat_starttime(stat), Some(12_345_678));
}
#[test]
fn proc_stat_starttime_parsing_rejects_truncated_lines() {
assert_eq!(parse_proc_stat_starttime(""), None);
assert_eq!(parse_proc_stat_starttime("1234 (svc)"), None);
assert_eq!(
parse_proc_stat_starttime("1234 (svc) S 1 1234 1234 0 -1 4194560 0"),
None
);
}
#[test]
fn zombie_and_reaped_states_count_as_dead() {
assert!(proc_stat_state_is_dead('Z'));
assert!(proc_stat_state_is_dead('X'));
assert!(proc_stat_state_is_dead('x'));
assert!(!proc_stat_state_is_dead('S'));
}
}
+438
View File
@@ -0,0 +1,438 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::cassandra::{apply_schema, config_from_env, verify_schema};
use crate::manifest::{API_PORT, DEV_PROXY_PORT, LOOPBACK_HOST};
use crate::proc::{RunOptions, merged_env, run_command, wait_http, wait_tcp};
use anyhow::{Context, Result, bail};
use futures_util::StreamExt;
use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderValue};
use serde_json::json;
use std::env;
use std::process::{Command, Output, Stdio};
use std::thread;
use std::time::{Duration, Instant};
use tokio::time::sleep;
use tokio_tungstenite::{connect_async, tungstenite::Message};
use url::Url;
pub const S3_BUCKETS: &[&str] = &[
"fluxer",
"fluxer-uploads",
"fluxer-downloads",
"fluxer-reports",
"fluxer-harvests",
"fluxer-static",
];
pub async fn run_smoke(quick: bool, public: bool) -> Result<()> {
let timeout = if quick { 5 } else { 120 };
wait_tcp(
"Valkey",
&env::var("VALKEY_HOST").unwrap_or_else(|_| "valkey".to_owned()),
env::var("VALKEY_PORT")
.unwrap_or_else(|_| "6379".to_owned())
.parse()?,
timeout,
)
.await?;
wait_tcp(
"NATS",
&env::var("NATS_HOST").unwrap_or_else(|_| "nats".to_owned()),
env::var("NATS_PORT")
.unwrap_or_else(|_| "4222".to_owned())
.parse()?,
timeout,
)
.await?;
wait_tcp(
"LiveKit",
&env::var("LIVEKIT_HOST").unwrap_or_else(|_| "livekit".to_owned()),
env::var("LIVEKIT_PORT")
.unwrap_or_else(|_| "7880".to_owned())
.parse()?,
timeout,
)
.await?;
wait_tcp(
"Mailpit SMTP",
&env::var("MAILPIT_HOST").unwrap_or_else(|_| "mailpit".to_owned()),
env::var("MAILPIT_SMTP_PORT")
.unwrap_or_else(|_| "1025".to_owned())
.parse()?,
timeout,
)
.await?;
wait_tcp(
"SeaweedFS S3",
&env::var("S3_HOST").unwrap_or_else(|_| "127.0.0.1".to_owned()),
env::var("S3_PORT")
.unwrap_or_else(|_| "8333".to_owned())
.parse()?,
timeout,
)
.await?;
wait_http(
"SeaweedFS master",
&env::var("SEAWEEDFS_MASTER_URL")
.unwrap_or_else(|_| "http://127.0.0.1:9333/cluster/status".to_owned()),
timeout,
)
.await?;
wait_s3_api(timeout).await?;
if !quick {
verify_schema(None, None).await?;
check_s3_buckets()?;
check_config_load()?;
}
if public {
check_gateway_internal_rpc().await?;
check_public_routes(timeout).await?;
}
println!("Fluxer smoke checks passed.");
Ok(())
}
fn check_config_load() -> Result<()> {
let script = concat!(
"void import('./packages/config/src/ConfigLoader.ts')",
".then(async (module) => { const config = await module.loadConfig(); ",
"if (!config.integrations.voice.enabled) throw new Error('voice disabled'); ",
"console.log('config: ok'); })",
".catch((error) => { console.error(error); process.exit(1); });",
);
crate::proc::run(&["pnpm", "exec", "tsx", "-e", script])
}
pub fn s3_endpoint() -> String {
env::var("FLUXER_S3_ENDPOINT").unwrap_or_else(|_| "http://127.0.0.1:8333".to_owned())
}
pub fn s3_env() -> Vec<(String, Option<String>)> {
vec![
(
"AWS_ACCESS_KEY_ID".to_owned(),
Some(env::var("FLUXER_S3_ACCESS_KEY_ID").unwrap_or_else(|_| "fluxer".to_owned())),
),
(
"AWS_SECRET_ACCESS_KEY".to_owned(),
Some(
env::var("FLUXER_S3_SECRET_ACCESS_KEY")
.unwrap_or_else(|_| "fluxer-secret".to_owned()),
),
),
(
"AWS_DEFAULT_REGION".to_owned(),
Some(env::var("FLUXER_S3_REGION").unwrap_or_else(|_| "us-east-1".to_owned())),
),
]
}
pub async fn wait_s3_api(timeout_secs: u64) -> Result<()> {
let endpoint = s3_endpoint();
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let mut last_output = String::new();
while Instant::now() < deadline {
let env = merged_env(Some(&s3_env()), true)?;
let output = Command::new("aws")
.args(["--endpoint-url", &endpoint, "s3api", "list-buckets"])
.env_clear()
.envs(env)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.context("failed to run aws s3api list-buckets")?;
if output.status.success() {
println!("SeaweedFS S3 API is reachable at {endpoint}");
return Ok(());
}
let mut combined = output.stdout;
combined.extend(output.stderr);
last_output = String::from_utf8_lossy(&combined).trim().to_owned();
sleep(Duration::from_secs(2)).await;
}
bail!("Timed out waiting for SeaweedFS S3 API at {endpoint}: {last_output}");
}
pub fn ensure_s3_buckets() -> Result<()> {
let endpoint = s3_endpoint();
let env = s3_env();
for bucket in S3_BUCKETS {
ensure_s3_bucket(&endpoint, &env, bucket)?;
}
Ok(())
}
fn ensure_s3_bucket(endpoint: &str, env: &[(String, Option<String>)], bucket: &str) -> Result<()> {
let deadline = Instant::now() + Duration::from_secs(30);
loop {
if s3_bucket_exists(endpoint, env, bucket)? {
println!("S3 bucket exists: {bucket}");
return Ok(());
}
let output = create_s3_bucket(endpoint, env, bucket)?;
if output.status.success() {
println!("Created S3 bucket: {bucket}");
return Ok(());
}
let text = command_output_text(&output);
if !s3_bucket_already_exists_output(&text) {
let code = output.status.code().unwrap_or(-1);
bail!(
"Command failed with exit code {code}: aws --endpoint-url {endpoint} s3api create-bucket --bucket {bucket}"
);
}
if Instant::now() >= deadline {
bail!(
"Timed out waiting for S3 bucket {bucket} to become readable after create-bucket reported it already exists: {text}"
);
}
thread::sleep(Duration::from_secs(1));
}
}
fn s3_bucket_exists(
endpoint: &str,
env: &[(String, Option<String>)],
bucket: &str,
) -> Result<bool> {
let output = run_command(
&[
"aws",
"--endpoint-url",
endpoint,
"s3api",
"head-bucket",
"--bucket",
bucket,
],
RunOptions {
env: env.to_vec(),
check: false,
capture: true,
..RunOptions::default()
},
)?;
Ok(output.status.success())
}
fn create_s3_bucket(
endpoint: &str,
env: &[(String, Option<String>)],
bucket: &str,
) -> Result<Output> {
run_command(
&[
"aws",
"--endpoint-url",
endpoint,
"s3api",
"create-bucket",
"--bucket",
bucket,
],
RunOptions {
env: env.to_vec(),
check: false,
capture: true,
..RunOptions::default()
},
)
}
fn command_output_text(output: &Output) -> String {
let mut combined = output.stdout.clone();
combined.extend_from_slice(&output.stderr);
String::from_utf8_lossy(&combined).trim().to_owned()
}
fn s3_bucket_already_exists_output(output: &str) -> bool {
output.contains("BucketAlreadyExists") || output.contains("BucketAlreadyOwnedByYou")
}
fn check_s3_buckets() -> Result<()> {
let endpoint = s3_endpoint();
let env = s3_env();
for bucket in S3_BUCKETS {
run_command(
&[
"aws",
"--endpoint-url",
&endpoint,
"s3api",
"head-bucket",
"--bucket",
bucket,
],
RunOptions {
env: env.clone(),
capture: true,
..RunOptions::default()
},
)?;
}
println!("S3 buckets: ok");
Ok(())
}
async fn check_public_routes(timeout_secs: u64) -> Result<()> {
let base_url = public_smoke_base_url();
wait_http(
"dev proxy api",
&format!("{base_url}/api/_health"),
timeout_secs,
)
.await?;
wait_http(
"dev proxy media",
&format!("{base_url}/media/_health"),
timeout_secs,
)
.await?;
wait_http(
"dev proxy gateway",
&format!("{base_url}/gateway/_health"),
timeout_secs,
)
.await?;
check_gateway_websocket(&base_url, timeout_secs).await?;
wait_http("dev proxy app", &format!("{base_url}/"), timeout_secs).await?;
wait_http("devmail", &format!("{base_url}/devmail/"), timeout_secs).await
}
async fn check_gateway_websocket(base_url: &str, timeout_secs: u64) -> Result<()> {
let url = gateway_websocket_url(base_url)?;
let connect_timeout = Duration::from_secs(timeout_secs.clamp(5, 30));
let (mut socket, _) = tokio::time::timeout(connect_timeout, connect_async(url.as_str()))
.await
.with_context(|| format!("timed out connecting to gateway websocket at {url}"))?
.with_context(|| format!("failed to connect to gateway websocket at {url}"))?;
let frame = tokio::time::timeout(Duration::from_secs(10), socket.next())
.await
.with_context(|| format!("timed out waiting for gateway websocket hello at {url}"))?
.ok_or_else(|| anyhow::anyhow!("gateway websocket ended before hello at {url}"))?
.with_context(|| format!("gateway websocket receive failed at {url}"))?;
if matches!(frame, Message::Close(_)) {
bail!("gateway websocket closed before hello at {url}");
}
println!("gateway websocket is reachable at {url}");
Ok(())
}
fn gateway_websocket_url(base_url: &str) -> Result<String> {
let mut url =
Url::parse(base_url).with_context(|| format!("invalid public URL: {base_url}"))?;
let scheme = match url.scheme() {
"https" => "wss",
"http" => "ws",
scheme => bail!("public URL must use http or https for websocket smoke: {scheme}"),
};
url.set_scheme(scheme)
.map_err(|_| anyhow::anyhow!("failed to set websocket scheme for {base_url}"))?;
url.set_path("/gateway");
url.set_query(Some("v=1&encoding=json&compress=zstd-stream&stream=1"));
url.set_fragment(None);
Ok(url.to_string())
}
fn public_smoke_base_url() -> String {
if let Ok(url) = env::var("FLUXER_DEV_PUBLIC_SMOKE_URL") {
let url = url.trim();
if !url.is_empty() {
return url.trim_end_matches('/').to_owned();
}
}
if let Ok(url) = env::var("FLUXER_PUBLIC_URL") {
let url = url.trim().trim_end_matches('/');
if !url.is_empty() && !matches!(url, "http://localhost:8088" | "http://127.0.0.1:8088") {
return url.to_owned();
}
}
if let Ok(url) = crate::tunnel::resolve_cloudflare_public_url(None) {
return url.trim_end_matches('/').to_owned();
}
format!("http://{LOOPBACK_HOST}:{DEV_PROXY_PORT}")
}
async fn check_gateway_internal_rpc() -> Result<()> {
let base_endpoint = env::var("FLUXER_INTERNAL_API_ENDPOINT")
.unwrap_or_else(|_| format!("http://{LOOPBACK_HOST}:{API_PORT}"));
let endpoint = env::var("FLUXER_GATEWAY_API_RPC_ENDPOINT")
.unwrap_or_else(|_| format!("{}/internal/rpc", base_endpoint.trim_end_matches('/')));
let token = env::var("FLUXER_GATEWAY_RPC_AUTH_TOKEN").unwrap_or_default();
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
headers.insert("x-forwarded-for", HeaderValue::from_static(LOOPBACK_HOST));
headers.insert("x-fluxer-rpc-auth", HeaderValue::from_str(&token)?);
let payload: serde_json::Value = reqwest::Client::new()
.post(endpoint)
.headers(headers)
.json(&json!({"type": "get_gateway_rollout_config"}))
.send()
.await?
.json()
.await?;
if payload.get("type").and_then(|value| value.as_str()) != Some("get_gateway_rollout_config")
|| payload.pointer("/data/config").is_none()
{
bail!("Gateway internal RPC returned unexpected payload: {payload}");
}
println!("Gateway internal RPC: ok");
Ok(())
}
pub async fn bootstrap_schema_and_object_store() -> Result<()> {
apply_schema(Some(config_from_env()?)).await?;
ensure_s3_buckets()
}
pub async fn http_ok(url: &str) -> bool {
reqwest::Client::new()
.get(url)
.timeout(Duration::from_secs(5))
.send()
.await
.map(|response| response.status().as_u16() < 500)
.unwrap_or(false)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn s3_env_uses_fluxer_defaults() {
let env = s3_env();
assert!(env.iter().any(
|(key, value)| key == "AWS_DEFAULT_REGION" && value.as_deref() == Some("us-east-1")
));
}
#[test]
fn bucket_already_exists_output_matches_aws_errors() {
assert!(s3_bucket_already_exists_output(
"An error occurred (BucketAlreadyExists) when calling the CreateBucket operation"
));
assert!(s3_bucket_already_exists_output(
"An error occurred (BucketAlreadyOwnedByYou) when calling the CreateBucket operation"
));
assert!(!s3_bucket_already_exists_output(
"An error occurred (AccessDenied) when calling the CreateBucket operation"
));
}
#[test]
fn gateway_websocket_url_uses_public_scheme() {
assert_eq!(
gateway_websocket_url("https://dev.example.test").unwrap(),
"wss://dev.example.test/gateway?v=1&encoding=json&compress=zstd-stream&stream=1"
);
assert_eq!(
gateway_websocket_url("http://localhost:8088").unwrap(),
"ws://localhost:8088/gateway?v=1&encoding=json&compress=zstd-stream&stream=1"
);
}
}
+132
View File
@@ -0,0 +1,132 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::desktop::build_desktop;
use crate::proc::{RunOptions, run_command};
use anyhow::Result;
use std::env;
use std::path::Path;
const DEFAULT_TEST_WORKSPACE_CONCURRENCY: &str = "2";
const DEFAULT_API_TEST_WORKERS: &str = "2";
fn task_run(args: &[&str]) -> Result<()> {
run_command(
args,
RunOptions {
load_default_env: false,
..RunOptions::default()
},
)
.map(drop)
}
fn test_env() -> Vec<(String, Option<String>)> {
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
let api_workers =
env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| DEFAULT_API_TEST_WORKERS.to_owned());
vec![
("FLUXER_NATS_URL".to_owned(), Some(nats_url.clone())),
(
"FLUXER_NATS_CORE_URL".to_owned(),
Some(env::var("FLUXER_NATS_CORE_URL").unwrap_or_else(|_| nats_url.clone())),
),
(
"FLUXER_NATS_JETSTREAM_URL".to_owned(),
Some(env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone())),
),
("API_TEST_MAX_WORKERS".to_owned(), Some(api_workers.clone())),
(
"API_TEST_MAX_CONCURRENCY".to_owned(),
Some(env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers)),
),
]
}
fn default_test_nats_url() -> String {
let host = if Path::new("/.dockerenv").exists() {
"nats"
} else {
"127.0.0.1"
};
format!("nats://{host}:4222")
}
fn run_generators(for_typecheck: bool) -> Result<()> {
task_run(&["pnpm", "--filter", "@fluxer/config", "generate"])?;
task_run(&["pnpm", "--filter", "@fluxer/schema", "generate"])?;
if for_typecheck {
task_run(&["pnpm", "--filter", "@fluxer/i18n", "generate:types"])?;
}
task_run(&["pnpm", "--filter", "fluxer_app", "i18n:compile"])
}
pub fn run_typecheck() -> Result<i32> {
run_generators(true)?;
task_run(&["pnpm", "-r", "--if-present", "typecheck"])?;
Ok(0)
}
pub fn run_test() -> Result<i32> {
run_generators(false)?;
let workspace_concurrency = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY")
.unwrap_or_else(|_| DEFAULT_TEST_WORKSPACE_CONCURRENCY.to_owned());
let env = test_env();
run_command(
&[
"pnpm",
"-r",
&format!("--workspace-concurrency={workspace_concurrency}"),
"--filter",
"!fluxer_api",
"--filter",
"!fluxer",
"--if-present",
"test",
],
RunOptions {
env: env.clone(),
load_default_env: false,
..RunOptions::default()
},
)?;
run_command(
&["pnpm", "--filter", "fluxer_api", "test"],
RunOptions {
env,
load_default_env: false,
..RunOptions::default()
},
)?;
Ok(0)
}
pub fn run_build() -> Result<i32> {
run_generators(false)?;
task_run(&["pnpm", "--filter", "fluxer_app", "build"])?;
build_desktop(false)?;
Ok(0)
}
pub fn run_knip() -> Result<i32> {
task_run(&["pnpm", "--filter", "fluxer_app", "i18n:compile"])?;
task_run(&["pnpm", "exec", "knip"])?;
Ok(0)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_nats_url_switches_inside_container() {
let expected_host = if Path::new("/.dockerenv").exists() {
"nats"
} else {
"127.0.0.1"
};
assert_eq!(
default_test_nats_url(),
format!("nats://{expected_host}:4222")
);
}
}
+469
View File
@@ -0,0 +1,469 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::manifest::{DEV_PROXY_PORT, LOCAL_APP_URL, LOOPBACK_HOST};
use crate::paths::{DEV_STATE_DIR, ROOT, which};
use crate::proc::wait_tcp;
use anyhow::{Context, Result, bail};
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use url::Url;
const LOCAL_ENV_START: &str = "# BEGIN fluxer-dev public URL";
const LOCAL_ENV_END: &str = "# END fluxer-dev public URL";
const DEFAULT_TOKEN_FILE_NAME: &str = "cloudflare-tunnel-token";
const DEFAULT_PUBLIC_URL_FILE_NAME: &str = "cloudflare-tunnel-public-url";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PublicUrl {
pub base_url: String,
pub host: String,
pub scheme: String,
pub public_port: u16,
pub websocket_scheme: String,
}
pub fn parse_public_url(raw: &str) -> Result<PublicUrl> {
let url = Url::parse(raw).with_context(|| format!("invalid public URL: {raw}"))?;
let scheme = url.scheme().to_owned();
if scheme != "http" && scheme != "https" {
bail!("Public URL must use http or https: {raw}");
}
let host = url
.host_str()
.ok_or_else(|| anyhow::anyhow!("Public URL must include a host: {raw}"))?
.to_owned();
let path = url.path();
if path != "/" && !path.is_empty() {
bail!("Public URL must not include a path: {raw}");
}
if url.query().is_some() || url.fragment().is_some() {
bail!("Public URL must not include a query string or fragment: {raw}");
}
let public_port = url
.port_or_known_default()
.ok_or_else(|| anyhow::anyhow!("Public URL has no known port: {raw}"))?;
let mut normalized = url;
normalized.set_path("");
normalized.set_query(None);
normalized.set_fragment(None);
Ok(PublicUrl {
base_url: normalized.as_str().trim_end_matches('/').to_owned(),
host,
scheme: scheme.clone(),
public_port,
websocket_scheme: if scheme == "https" { "wss" } else { "ws" }.to_owned(),
})
}
pub fn public_url_env(public_url: &str) -> Result<Vec<(String, String)>> {
let parsed = parse_public_url(public_url)?;
let base = parsed.base_url.as_str();
let gateway_base = format!("{}://{}", parsed.websocket_scheme, parsed.host);
let localhost_origins = "http://localhost,http://localhost:8088";
Ok(vec![
("FLUXER_BASE_DOMAIN".to_owned(), parsed.host.clone()),
("FLUXER_PUBLIC_SCHEME".to_owned(), parsed.scheme.clone()),
(
"FLUXER_PUBLIC_PORT".to_owned(),
parsed.public_port.to_string(),
),
("FLUXER_STATIC_CDN_DOMAIN".to_owned(), parsed.host.clone()),
("FLUXER_PUBLIC_URL".to_owned(), base.to_owned()),
("FLUXER_API_ENDPOINT".to_owned(), format!("{base}/api")),
(
"FLUXER_API_CLIENT_ENDPOINT".to_owned(),
format!("{base}/api"),
),
("FLUXER_APP_ENDPOINT".to_owned(), base.to_owned()),
(
"FLUXER_GATEWAY_ENDPOINT".to_owned(),
format!("{gateway_base}/gateway"),
),
("FLUXER_MEDIA_ENDPOINT".to_owned(), format!("{base}/media")),
("FLUXER_S3_PUBLIC_ENDPOINT".to_owned(), base.to_owned()),
("FLUXER_S3_FORCE_PATH_STYLE".to_owned(), "true".to_owned()),
("FLUXER_STATIC_CDN_ENDPOINT".to_owned(), base.to_owned()),
("FLUXER_ADMIN_ENDPOINT".to_owned(), format!("{base}/admin")),
(
"FLUXER_MARKETING_ENDPOINT".to_owned(),
format!("{base}/marketing"),
),
(
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT".to_owned(),
format!("{base}/media"),
),
(
"FLUXER_LIVEKIT_URL".to_owned(),
format!("{gateway_base}/livekit"),
),
(
"FLUXER_LIVEKIT_WEBHOOK_URL".to_owned(),
format!("{base}/api/webhooks/livekit"),
),
(
"FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT".to_owned(),
format!("{base}/media"),
),
(
"FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT".to_owned(),
format!("{base}/media"),
),
(
"FLUXER_GATEWAY_STATIC_CDN_ENDPOINT".to_owned(),
base.to_owned(),
),
(
"FLUXER_ADMIN_OAUTH_REDIRECT_URI".to_owned(),
format!("{base}/admin/oauth2_callback"),
),
("FLUXER_PASSKEY_RP_ID".to_owned(), parsed.host.clone()),
(
"FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS".to_owned(),
format!("{localhost_origins},{base}"),
),
(
"PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT".to_owned(),
format!("{base}/api"),
),
])
}
pub fn apply_public_url_env(public_url: &str) -> Result<()> {
for (key, value) in public_url_env(public_url)? {
unsafe {
std::env::set_var(key, value);
}
}
Ok(())
}
pub fn public_url_env_text(public_url: &str) -> Result<String> {
let mut text = String::new();
text.push_str(LOCAL_ENV_START);
text.push('\n');
text.push_str("# Generated by `fluxer-dev tunnel configure`; safe to delete.\n");
for (key, value) in public_url_env(public_url)? {
text.push_str(&key);
text.push('=');
text.push_str(&value);
text.push('\n');
}
text.push_str(LOCAL_ENV_END);
text.push('\n');
Ok(text)
}
pub fn write_public_url_local_env(path: &Path, public_url: &str) -> Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
}
let generated = public_url_env_text(public_url)?;
let existing = fs::read_to_string(path).unwrap_or_default();
let next = replace_marked_block(&existing, &generated);
fs::write(path, next).with_context(|| format!("failed to write {}", path.display()))?;
println!("Wrote public URL overrides to {}", path.display());
Ok(())
}
pub fn default_token_file() -> PathBuf {
DEV_STATE_DIR.join(DEFAULT_TOKEN_FILE_NAME)
}
pub fn default_public_url_file() -> PathBuf {
DEV_STATE_DIR.join(DEFAULT_PUBLIC_URL_FILE_NAME)
}
pub fn write_cloudflare_public_url_file(public_url: &str) -> Result<PathBuf> {
fs::create_dir_all(DEV_STATE_DIR.as_path())
.with_context(|| format!("failed to create {}", DEV_STATE_DIR.display()))?;
let parsed = parse_public_url(public_url)?;
let path = default_public_url_file();
fs::write(&path, format!("{}\n", parsed.base_url))
.with_context(|| format!("failed to write {}", path.display()))?;
println!("Saved Cloudflare tunnel public URL to {}", path.display());
Ok(path)
}
pub fn write_cloudflare_token_file(token: &str) -> Result<PathBuf> {
fs::create_dir_all(DEV_STATE_DIR.as_path())
.with_context(|| format!("failed to create {}", DEV_STATE_DIR.display()))?;
let path = default_token_file();
fs::write(&path, format!("{}\n", token.trim()))
.with_context(|| format!("failed to write {}", path.display()))?;
set_private_file_mode(&path)?;
println!("Saved Cloudflare tunnel token to {}", path.display());
Ok(path)
}
pub fn resolve_cloudflare_public_url(public_url_arg: Option<&str>) -> Result<String> {
if let Some(public_url) = public_url_arg {
return Ok(parse_public_url(public_url)?.base_url);
}
for key in [
"FLUXER_CLOUDFLARE_TUNNEL_PUBLIC_URL",
"CLOUDFLARE_TUNNEL_PUBLIC_URL",
] {
if let Ok(public_url) = std::env::var(key) {
let public_url = public_url.trim();
if !public_url.is_empty() {
return Ok(parse_public_url(public_url)?.base_url);
}
}
}
if let Ok(public_url) = std::env::var("FLUXER_PUBLIC_URL") {
let public_url = public_url.trim();
if !public_url.is_empty() && public_url != LOCAL_APP_URL {
return Ok(parse_public_url(public_url)?.base_url);
}
}
let path = default_public_url_file();
if let Ok(public_url) = fs::read_to_string(&path) {
let public_url = public_url.trim();
if !public_url.is_empty() {
return Ok(parse_public_url(public_url)?.base_url);
}
}
bail!(
"Missing Cloudflare tunnel public URL. Run `pnpm dev:tunnel:configure -- --public-url https://...` or pass `pnpm dev -- --cloudflare-tunnel --public-url https://...`."
);
}
pub fn apply_cloudflare_public_url_env(public_url_arg: Option<&str>) -> Result<String> {
let public_url = resolve_cloudflare_public_url(public_url_arg)?;
apply_public_url_env(&public_url)?;
Ok(public_url)
}
pub async fn run_cloudflare_tunnel(
token_arg: Option<String>,
token_file: Option<PathBuf>,
) -> Result<i32> {
let token = resolve_cloudflare_token(token_arg, token_file.as_deref())?;
wait_cloudflare_tunnel_origin().await?;
let public_url = display_public_url();
println!("Starting Cloudflare Tunnel for {public_url} -> http://127.0.0.1:8088");
if let Some(binary) = resolve_cloudflared_binary() {
let status = Command::new(binary)
.args(["tunnel", "run", "--token", token.as_str()])
.status()
.context("failed to start cloudflared")?;
return Ok(status.code().unwrap_or(1));
}
if running_inside_devcontainer() {
let docker = docker_command();
let mut command = Command::new(&docker[0]);
command.args(&docker[1..]);
let status = command
.args([
"run",
"--rm",
"--network",
&format!(
"container:{}",
std::env::var("HOSTNAME")
.unwrap_or_else(|_| "fluxer-dev-workspace-1".to_owned())
),
"cloudflare/cloudflared:latest",
"tunnel",
"run",
"--token",
token.as_str(),
])
.status()
.context("failed to start cloudflare/cloudflared Docker image")?;
return Ok(status.code().unwrap_or(1));
}
bail!(
"cloudflared is not installed. Install cloudflared or run from the devcontainer with Docker available."
);
}
async fn wait_cloudflare_tunnel_origin() -> Result<()> {
let timeout = std::env::var("FLUXER_CLOUDFLARE_TUNNEL_ORIGIN_READY_TIMEOUT")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(30);
if let Err(error) = wait_tcp("Fluxer dev proxy", LOOPBACK_HOST, DEV_PROXY_PORT, timeout).await {
bail!(
"Cloudflare Tunnel origin is not reachable at http://{LOOPBACK_HOST}:{DEV_PROXY_PORT}: {error}\nStart the full stack with `pnpm dev:tunnel`; `pnpm dev:tunnel:run` only starts cloudflared and expects the dev proxy to already be running."
);
}
Ok(())
}
fn resolve_cloudflare_token(
token_arg: Option<String>,
token_file: Option<&Path>,
) -> Result<String> {
if let Some(token) = token_arg {
let token = token.trim();
if !token.is_empty() {
return Ok(token.to_owned());
}
}
for key in [
"FLUXER_CLOUDFLARE_TUNNEL_TOKEN",
"CLOUDFLARE_TUNNEL_TOKEN",
"TUNNEL_TOKEN",
] {
if let Ok(token) = std::env::var(key) {
let token = token.trim();
if !token.is_empty() {
return Ok(token.to_owned());
}
}
}
let path = token_file
.map(Path::to_path_buf)
.unwrap_or_else(default_token_file);
if let Ok(token) = fs::read_to_string(&path) {
let token = token.trim();
if !token.is_empty() {
return Ok(token.to_owned());
}
}
bail!(
"Missing Cloudflare tunnel token. Set FLUXER_CLOUDFLARE_TUNNEL_TOKEN or write {}.",
path.display()
);
}
fn display_public_url() -> String {
if let Ok(public_url) = std::env::var("FLUXER_PUBLIC_URL") {
let public_url = public_url.trim();
if !public_url.is_empty() && public_url != LOCAL_APP_URL {
return public_url.to_owned();
}
}
if let Ok(public_url) = fs::read_to_string(default_public_url_file()) {
let public_url = public_url.trim();
if !public_url.is_empty() {
return public_url.to_owned();
}
}
LOCAL_APP_URL.to_owned()
}
fn resolve_cloudflared_binary() -> Option<PathBuf> {
if let Ok(binary) = std::env::var("FLUXER_CLOUDFLARED_BIN")
&& !binary.trim().is_empty()
{
return Some(PathBuf::from(binary));
}
which("cloudflared")
}
fn running_inside_devcontainer() -> bool {
ROOT.starts_with("/workspaces")
&& Path::new("/var/run/docker.sock").exists()
&& which("docker").is_some()
}
fn docker_command() -> Vec<PathBuf> {
if docker_socket_is_writable() || which("sudo").is_none() {
return vec![PathBuf::from("docker")];
}
vec![PathBuf::from("sudo"), PathBuf::from("docker")]
}
fn docker_socket_is_writable() -> bool {
std::os::unix::net::UnixStream::connect("/var/run/docker.sock").is_ok()
}
fn replace_marked_block(existing: &str, generated: &str) -> String {
let Some(start) = existing.find(LOCAL_ENV_START) else {
return append_block(existing, generated);
};
let Some(relative_end) = existing[start..].find(LOCAL_ENV_END) else {
return append_block(existing, generated);
};
let end = start + relative_end + LOCAL_ENV_END.len();
let mut next = String::new();
next.push_str(existing[..start].trim_end());
if !next.is_empty() {
next.push_str("\n\n");
}
next.push_str(generated.trim_end());
let suffix = existing[end..].trim_start_matches(['\r', '\n']);
if !suffix.is_empty() {
next.push_str("\n\n");
next.push_str(suffix);
} else {
next.push('\n');
}
next
}
fn append_block(existing: &str, generated: &str) -> String {
let mut next = existing.trim_end().to_owned();
if !next.is_empty() {
next.push_str("\n\n");
}
next.push_str(generated.trim_end());
next.push('\n');
next
}
#[cfg(unix)]
fn set_private_file_mode(path: &Path) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
let mut permissions = fs::metadata(path)?.permissions();
permissions.set_mode(0o600);
fs::set_permissions(path, permissions)?;
Ok(())
}
#[cfg(not(unix))]
fn set_private_file_mode(_path: &Path) -> Result<()> {
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn derives_https_public_url_env() {
let env = public_url_env("https://dev.example.test").unwrap();
assert!(env.iter().any(|(key, value)| {
key == "FLUXER_GATEWAY_ENDPOINT" && value == "wss://dev.example.test/gateway"
}));
assert!(env.iter().any(|(key, value)| {
key == "FLUXER_LIVEKIT_URL" && value == "wss://dev.example.test/livekit"
}));
assert!(env.iter().any(|(key, value)| {
key == "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT"
&& value == "https://dev.example.test/media"
}));
assert!(env.iter().any(|(key, value)| {
key == "FLUXER_S3_PUBLIC_ENDPOINT" && value == "https://dev.example.test"
}));
assert!(
env.iter()
.any(|(key, value)| { key == "FLUXER_PUBLIC_PORT" && value == "443" })
);
}
#[test]
fn rejects_public_url_with_path() {
assert!(parse_public_url("https://example.com/app").is_err());
}
#[test]
fn resolves_public_url_arg_normalized() {
assert_eq!(
resolve_cloudflare_public_url(Some("https://example.com/")).unwrap(),
"https://example.com"
);
}
#[test]
fn replaces_existing_generated_block() {
let existing =
"A=1\n\n# BEGIN fluxer-dev public URL\nOLD=1\n# END fluxer-dev public URL\n\nB=2\n";
let next = replace_marked_block(existing, "NEW=1\n");
assert_eq!(next, "A=1\n\nNEW=1\n\nB=2\n");
}
}
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "fluxer-i18n-auto"
version = "0.1.0"
edition.workspace = true
license.workspace = true
[dependencies]
anyhow = "1.0.102"
clap = { version = "4.6.1", features = ["derive"] }
regex = "1.12"
reqwest = { version = "0.13.4", default-features = false, features = ["blocking", "json", "rustls"] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.150"
[dev-dependencies]
tempfile = "3.27.0"
+212
View File
@@ -0,0 +1,212 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::fs;
use std::path::PathBuf;
use anyhow::Result;
use crate::config::{
DEFAULT_OPENROUTER_APP_TITLE, DEFAULT_OPENROUTER_BASE_URL, DEFAULT_OPENROUTER_FALLBACK_MODELS,
DEFAULT_OPENROUTER_HTTP_REFERER, DEFAULT_OPENROUTER_MODEL, DEFAULT_OPENROUTER_PROVIDER_SORT,
EnvOverlay, env_value, trim_trailing_slash,
};
use crate::openrouter::is_openrouter_available;
use crate::runner::translate_main;
const ENV_KEYS: &[&str] = &[
"FLUXER_AUTO_I18N",
"FLUXER_AUTO_I18N_LOCALE_CONCURRENCY",
"FLUXER_AUTO_I18N_PROGRESS_INTERVAL",
"FLUXER_AUTO_I18N_REQUEST_TIMEOUT",
"FLUXER_AUTO_I18N_STRING_CONCURRENCY",
"I18N_LLM_MODEL",
"OPENROUTER_API_KEY",
"OPENROUTER_APP_TITLE",
"OPENROUTER_BASE_URL",
"OPENROUTER_FALLBACK_MODELS",
"OPENROUTER_HTTP_REFERER",
"OPENROUTER_MODEL",
"OPENROUTER_PROVIDER_SORT",
];
pub fn auto_main(args: &[String]) -> Result<u8> {
let env_overrides = load_env_from_files(ENV_KEYS);
let fluxer_auto_i18n = env_value("FLUXER_AUTO_I18N", &env_overrides, "");
let openrouter_base_url = trim_trailing_slash(&env_value(
"OPENROUTER_BASE_URL",
&env_overrides,
DEFAULT_OPENROUTER_BASE_URL,
));
let openrouter_api_key = env_value("OPENROUTER_API_KEY", &env_overrides, "");
let i18n_llm_model = env_value("I18N_LLM_MODEL", &env_overrides, "")
.if_empty(|| env_value("OPENROUTER_MODEL", &env_overrides, DEFAULT_OPENROUTER_MODEL));
let bypass_run_gate = args.iter().any(|arg| {
matches!(arg.as_str(), "--self-test" | "--help" | "-h" | "--dry-run")
|| arg.starts_with("--dry-run=")
});
let explicitly_disabled = matches!(
fluxer_auto_i18n.to_lowercase().as_str(),
"0" | "false" | "no" | "off"
);
if explicitly_disabled && !bypass_run_gate {
eprintln!("i18n:auto skipped: FLUXER_AUTO_I18N=0 disables automatic translations.");
return Ok(0);
}
let openrouter_is_available =
is_openrouter_available(&openrouter_base_url, &openrouter_api_key);
let should_run = fluxer_auto_i18n == "1" || (!explicitly_disabled && openrouter_is_available);
if !should_run && !bypass_run_gate {
eprintln!(
"i18n:auto skipped: OpenRouter is unavailable. Set OPENROUTER_API_KEY, or set FLUXER_AUTO_I18N=1 to attempt translations with {} at {}.",
if i18n_llm_model.is_empty() {
DEFAULT_OPENROUTER_MODEL
} else {
&i18n_llm_model
},
openrouter_base_url
);
return Ok(0);
}
if should_run && fluxer_auto_i18n != "1" && openrouter_is_available {
println!(
"i18n:auto detected OpenRouter availability at {openrouter_base_url}; running without FLUXER_AUTO_I18N=1."
);
}
let mut runner_env = env_overrides;
runner_env.insert(
"FLUXER_AUTO_I18N".to_string(),
if should_run {
"1".to_string()
} else {
fluxer_auto_i18n
},
);
runner_env.insert("I18N_LLM_MODEL".to_string(), i18n_llm_model.clone());
runner_env.insert(
"OPENROUTER_MODEL".to_string(),
env_value("OPENROUTER_MODEL", &runner_env, DEFAULT_OPENROUTER_MODEL),
);
runner_env.insert("OPENROUTER_BASE_URL".to_string(), openrouter_base_url);
runner_env.insert("OPENROUTER_API_KEY".to_string(), openrouter_api_key);
runner_env.insert(
"OPENROUTER_FALLBACK_MODELS".to_string(),
env_value(
"OPENROUTER_FALLBACK_MODELS",
&runner_env,
DEFAULT_OPENROUTER_FALLBACK_MODELS,
),
);
runner_env.insert(
"OPENROUTER_PROVIDER_SORT".to_string(),
env_value(
"OPENROUTER_PROVIDER_SORT",
&runner_env,
DEFAULT_OPENROUTER_PROVIDER_SORT,
),
);
runner_env.insert(
"OPENROUTER_HTTP_REFERER".to_string(),
env_value(
"OPENROUTER_HTTP_REFERER",
&runner_env,
DEFAULT_OPENROUTER_HTTP_REFERER,
),
);
runner_env.insert(
"OPENROUTER_APP_TITLE".to_string(),
env_value(
"OPENROUTER_APP_TITLE",
&runner_env,
DEFAULT_OPENROUTER_APP_TITLE,
),
);
translate_main(args, &runner_env)
}
pub fn load_env_from_files(keys: &[&str]) -> EnvOverlay {
let target_keys = keys
.iter()
.copied()
.collect::<std::collections::HashSet<_>>();
let mut env = EnvOverlay::new();
let Some(home) = std::env::var_os("HOME").map(PathBuf::from) else {
return env;
};
for candidate in [
".bash_profile",
".bashrc",
".profile",
".zprofile",
".zshrc",
] {
let file_path = home.join(candidate);
let Ok(content) = fs::read_to_string(file_path) else {
continue;
};
for line in content.lines() {
let Some((key, value)) = parse_export_line(line) else {
continue;
};
if target_keys.contains(key.as_str()) {
env.entry(key).or_insert(value);
}
}
}
env
}
pub fn parse_export_line(line: &str) -> Option<(String, String)> {
let trimmed = line.trim();
let key_value = trimmed.strip_prefix("export ")?;
let (key, value) = key_value.split_once('=')?;
let mut chars = key.chars();
let first = chars.next()?;
if !(first.is_alphabetic() || first == '_') {
return None;
}
if !chars.all(|character| character.is_alphanumeric() || character == '_') {
return None;
}
Some((key.to_string(), strip_quotes(value)))
}
pub fn strip_quotes(value: &str) -> String {
let trimmed = value.trim();
let first = trimmed.chars().next();
let last = trimmed.chars().next_back();
if trimmed.len() >= 2 && first == last && matches!(first, Some('\'' | '"')) {
trimmed[1..trimmed.len() - 1].to_string()
} else {
trimmed.to_string()
}
}
trait EmptyFallback {
fn if_empty(self, fallback: impl FnOnce() -> String) -> String;
}
impl EmptyFallback for String {
fn if_empty(self, fallback: impl FnOnce() -> String) -> String {
if self.is_empty() { fallback() } else { self }
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_simple_export_lines() {
assert_eq!(
parse_export_line(" export OPENROUTER_MODEL=\"translator\" "),
Some(("OPENROUTER_MODEL".to_string(), "translator".to_string()))
);
assert_eq!(
parse_export_line("export FLUXER_AUTO_I18N=1"),
Some(("FLUXER_AUTO_I18N".to_string(), "1".to_string()))
);
assert_eq!(parse_export_line("OPENROUTER_MODEL=translator"), None);
assert_eq!(parse_export_line("export 1BAD=value"), None);
}
}
+80
View File
@@ -0,0 +1,80 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::collections::HashMap;
use std::path::{Path, PathBuf};
pub const SOURCE_LOCALE: &str = "en-US";
pub const DEFAULT_OPENROUTER_BASE_URL: &str = "https://openrouter.ai/api/v1";
pub const DEFAULT_OPENROUTER_MODEL: &str = "google/gemini-2.5-flash-lite";
pub const DEFAULT_OPENROUTER_FALLBACK_MODELS: &str =
"openai/gpt-4.1-nano,mistralai/mistral-nemo,mistralai/mistral-small-3.2-24b-instruct";
pub const DEFAULT_OPENROUTER_PROVIDER_SORT: &str = "throughput";
pub const DEFAULT_OPENROUTER_HTTP_REFERER: &str = "https://fluxer.chat";
pub const DEFAULT_OPENROUTER_APP_TITLE: &str = "Fluxer i18n auto";
pub const DEFAULT_STRING_CONCURRENCY: usize = 2;
pub const DEFAULT_LOCALE_CONCURRENCY: usize = 1;
pub const DEFAULT_REQUEST_TIMEOUT_SECONDS: f64 = 300.0;
pub const DEFAULT_PROGRESS_INTERVAL_SECONDS: f64 = 20.0;
pub const GUIDANCE_EXCERPT_CHAR_LIMIT: usize = 1600;
pub const AUTO_I18N_UNCHANGED_COMMENT: &str = "# auto-i18n: reviewed unchanged";
pub const AUTO_I18N_LEGACY_UNCHANGED_COMMENT: &str = "#. auto-i18n: reviewed unchanged";
pub const AUTO_I18N_COMMENT_PREFIX: &str = "auto-i18n:";
pub const AUTO_I18N_REVIEWED_UNCHANGED_FILE: &str = "auto-i18n-reviewed-unchanged.json";
pub fn is_auto_i18n_unchanged_comment(comment: &str) -> bool {
let text = comment
.trim()
.strip_prefix("#. ")
.or_else(|| comment.trim().strip_prefix("# "))
.unwrap_or_else(|| comment.trim())
.trim();
text == "auto-i18n: reviewed unchanged"
}
pub type EnvOverlay = HashMap<String, String>;
pub fn default_app_dir() -> PathBuf {
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
manifest_dir
.parent()
.and_then(Path::parent)
.map(|repo_root| repo_root.join("fluxer_app"))
.unwrap_or_else(|| PathBuf::from("fluxer_app"))
}
pub fn i18n_dir(app_dir: &Path) -> PathBuf {
app_dir.join("src").join("features").join("i18n")
}
pub fn locales_dir(app_dir: &Path) -> PathBuf {
i18n_dir(app_dir).join("locales")
}
pub fn env_value(key: &str, env_overrides: &EnvOverlay, fallback: &str) -> String {
std::env::var(key)
.ok()
.filter(|value| !value.is_empty())
.or_else(|| env_overrides.get(key).cloned())
.unwrap_or_else(|| fallback.to_string())
}
pub fn positive_float_env(key: &str, fallback: f64, env_overrides: &EnvOverlay) -> f64 {
let value = env_value(key, env_overrides, "");
if value.is_empty() {
return fallback;
}
value
.parse::<f64>()
.ok()
.filter(|parsed| *parsed > 0.0)
.unwrap_or(fallback)
}
pub fn positive_int_env(key: &str, fallback: usize, env_overrides: &EnvOverlay) -> usize {
positive_float_env(key, fallback as f64, env_overrides).max(1.0) as usize
}
pub fn trim_trailing_slash(value: &str) -> String {
value.trim_end_matches('/').to_string()
}
+13
View File
@@ -0,0 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
pub mod auto;
pub mod config;
pub mod llm;
pub mod locales;
pub mod openrouter;
pub mod po;
pub mod prompts;
pub mod reviewed_unchanged;
pub mod runner;
pub mod tokens;
pub mod ts_catalog;
+114
View File
@@ -0,0 +1,114 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::Result;
use regex::Regex;
use serde_json::{Map, Value, json};
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LocalizationResult {
pub localized: String,
pub notes: String,
}
impl LocalizationResult {
pub fn localized(localized: impl Into<String>) -> Self {
Self {
localized: localized.into(),
notes: String::new(),
}
}
}
pub trait LocalizationClient: Sync {
fn base_url(&self) -> &str;
fn model(&self) -> &str;
fn request_timeout_seconds(&self) -> f64;
fn localize(
&self,
options: &Map<String, Value>,
system_prompt: &str,
user_prompt: &str,
) -> Result<LocalizationResult>;
}
pub fn base_options() -> Map<String, Value> {
let mut options = Map::new();
options.insert("temperature".to_string(), json!(0.1));
options.insert("top_p".to_string(), json!(0.9));
options.insert("num_ctx".to_string(), json!(4096));
options.insert("num_predict".to_string(), json!(800));
options
}
pub fn clean_translation_response(content: &str) -> String {
let trimmed = trim_special_response_tokens(content.trim());
let fenced_re = Regex::new(r"(?is)^```(?:[a-z]+)?\s*([\s\S]*?)\s*```$")
.expect("valid fenced response regex");
let mut raw = fenced_re
.captures(trimmed)
.and_then(|captures| captures.get(1))
.map(|matched| matched.as_str().trim().to_string())
.unwrap_or_else(|| trimmed.to_string());
let label_re = Regex::new(r"(?is)^(?:localized(?: string)?|translation|answer):\s*([\s\S]+)$")
.expect("valid label response regex");
if let Some(captures) = label_re.captures(&raw)
&& let Some(value) = captures.get(1)
{
raw = value.as_str().trim().to_string();
}
if raw.starts_with('{')
&& raw.ends_with('}')
&& let Ok(Value::Object(object)) = serde_json::from_str::<Value>(&raw)
&& let Some(localized) = object.get("localized").and_then(Value::as_str)
{
return localized.to_string();
}
if raw.len() >= 2
&& raw.starts_with('"')
&& raw.ends_with('"')
&& let Ok(Value::String(value)) = serde_json::from_str::<Value>(&raw)
{
return value;
}
raw
}
fn trim_special_response_tokens(content: &str) -> &str {
let mut trimmed = content.trim();
loop {
let next = trimmed
.strip_suffix("<|im_end|>")
.or_else(|| trimmed.strip_suffix("<end_of_turn>"))
.or_else(|| trimmed.strip_suffix("</s>"));
let Some(next) = next else {
return trimmed.trim();
};
trimmed = next.trim();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cleans_common_llm_response_wrappers() {
assert_eq!(
clean_translation_response("Translation: Bonjour"),
"Bonjour"
);
assert_eq!(clean_translation_response("\"Bonjour\""), "Bonjour");
assert_eq!(
clean_translation_response("{\"localized\":\"Bonjour\"}"),
"Bonjour"
);
assert_eq!(
clean_translation_response("```text\nBonjour\n```"),
"Bonjour"
);
assert_eq!(
clean_translation_response("```json\n{\"0\":\"bonjour\"}\n```<|im_end|>"),
"{\"0\":\"bonjour\"}"
);
}
}
+50
View File
@@ -0,0 +1,50 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
pub const LOCALE_DISPLAY_NAMES: &[(&str, &str)] = &[
("ar", "Arabic"),
("bg", "Bulgarian"),
("cs", "Czech"),
("da", "Danish"),
("de", "German"),
("el", "Greek"),
("en-GB", "English (United Kingdom)"),
("es-419", "Spanish (Latin America)"),
("es-ES", "Spanish (Spain)"),
("fi", "Finnish"),
("fr", "French"),
("he", "Hebrew"),
("hi", "Hindi"),
("hr", "Croatian"),
("hu", "Hungarian"),
("id", "Indonesian"),
("it", "Italian"),
("ja", "Japanese"),
("ko", "Korean"),
("lt", "Lithuanian"),
("nl", "Dutch"),
("no", "Norwegian Bokmal"),
("pl", "Polish"),
("pt-BR", "Portuguese (Brazil)"),
("ro", "Romanian"),
("ru", "Russian"),
("sv-SE", "Swedish (Sweden)"),
("th", "Thai"),
("tr", "Turkish"),
("uk", "Ukrainian"),
("vi", "Vietnamese"),
("zh-CN", "Simplified Chinese (Mainland China)"),
("zh-TW", "Traditional Chinese (Taiwan)"),
];
pub fn display_name(locale: &str) -> &str {
LOCALE_DISPLAY_NAMES
.iter()
.find_map(|(candidate, name)| (*candidate == locale).then_some(*name))
.unwrap_or(locale)
}
pub fn is_supported_locale(locale: &str) -> bool {
LOCALE_DISPLAY_NAMES
.iter()
.any(|(candidate, _name)| *candidate == locale)
}
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::process::ExitCode;
fn main() -> ExitCode {
let args = std::env::args().skip(1).collect::<Vec<_>>();
match fluxer_i18n_auto::auto::auto_main(&args) {
Ok(code) => ExitCode::from(code),
Err(error) => {
eprintln!("{error:#}");
ExitCode::from(1)
}
}
}
+257
View File
@@ -0,0 +1,257 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::time::Duration;
use anyhow::{Context, Result, bail};
use serde_json::{Map, Value, json};
use crate::llm::{LocalizationClient, LocalizationResult, clean_translation_response};
pub struct OpenRouterClient {
base_url: String,
model: String,
fallback_models: Vec<String>,
provider_sort: String,
http_referer: String,
app_title: String,
api_key: String,
request_timeout: Duration,
client: reqwest::blocking::Client,
}
#[derive(Clone, Debug, PartialEq)]
pub struct OpenRouterClientConfig {
pub base_url: String,
pub model: String,
pub fallback_models: Vec<String>,
pub provider_sort: String,
pub http_referer: String,
pub app_title: String,
pub api_key: String,
pub request_timeout_seconds: f64,
}
impl OpenRouterClient {
pub fn new(config: OpenRouterClientConfig) -> Result<Self> {
if config.api_key.trim().is_empty() {
bail!("OPENROUTER_API_KEY is required for i18n:auto");
}
let request_timeout = Duration::from_secs_f64(config.request_timeout_seconds);
let client = reqwest::blocking::Client::builder()
.timeout(request_timeout)
.build()
.context("failed to build OpenRouter HTTP client")?;
Ok(Self {
base_url: config.base_url.trim_end_matches('/').to_string(),
model: config.model,
fallback_models: config.fallback_models,
provider_sort: config.provider_sort,
http_referer: config.http_referer,
app_title: config.app_title,
api_key: config.api_key,
request_timeout,
client,
})
}
}
impl LocalizationClient for OpenRouterClient {
fn base_url(&self) -> &str {
&self.base_url
}
fn model(&self) -> &str {
&self.model
}
fn request_timeout_seconds(&self) -> f64 {
self.request_timeout.as_secs_f64()
}
fn localize(
&self,
options: &Map<String, Value>,
system_prompt: &str,
user_prompt: &str,
) -> Result<LocalizationResult> {
let max_tokens = options
.get("num_predict")
.and_then(Value::as_u64)
.unwrap_or(800);
let expects_json = system_prompt.contains("Return only valid compact JSON");
let mut provider = Map::new();
if !self.provider_sort.is_empty() {
provider.insert("sort".to_string(), json!(self.provider_sort));
}
provider.insert("allow_fallbacks".to_string(), json!(true));
if expects_json {
provider.insert("require_parameters".to_string(), json!(true));
}
let mut payload = Map::new();
payload.insert("model".to_string(), json!(self.model));
let fallback_models = self
.fallback_models
.iter()
.map(|model| model.trim())
.filter(|model| !model.is_empty() && *model != self.model)
.map(|model| json!(model))
.take(3)
.collect::<Vec<_>>();
if !fallback_models.is_empty() {
payload.insert("models".to_string(), Value::Array(fallback_models));
}
payload.insert(
"messages".to_string(),
json!([
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
]),
);
payload.insert(
"temperature".to_string(),
json!(
options
.get("temperature")
.and_then(Value::as_f64)
.unwrap_or(0.1)
),
);
payload.insert(
"top_p".to_string(),
json!(options.get("top_p").and_then(Value::as_f64).unwrap_or(0.9)),
);
payload.insert("max_tokens".to_string(), json!(max_tokens));
payload.insert("stream".to_string(), json!(false));
payload.insert("provider".to_string(), Value::Object(provider));
if expects_json {
payload.insert(
"response_format".to_string(),
json!({"type": "json_object"}),
);
}
let response = self
.client
.post(format!("{}/chat/completions", self.base_url))
.bearer_auth(&self.api_key)
.header("HTTP-Referer", &self.http_referer)
.header("X-OpenRouter-Title", &self.app_title)
.json(&Value::Object(payload))
.send()
.with_context(|| {
format!(
"OpenRouter request failed after {}s",
self.request_timeout.as_secs_f64()
)
})?;
let status = response.status();
let body = response
.text()
.context("failed to read OpenRouter response")?;
if !status.is_success() {
bail!(
"OpenRouter API error: {} - {}",
status.as_u16(),
truncate_error_body(&body)
);
}
let data = serde_json::from_str::<Value>(&body)
.context("failed to parse OpenRouter response JSON")?;
let Some(content) =
extract_chat_message_content(&data).filter(|content| !content.is_empty())
else {
bail!("Empty response from OpenRouter");
};
Ok(LocalizationResult::localized(clean_translation_response(
&content,
)))
}
}
pub fn parse_openrouter_fallback_models(value: &str) -> Vec<String> {
value
.split(',')
.map(str::trim)
.filter(|model| !model.is_empty())
.map(ToString::to_string)
.collect()
}
pub fn is_openrouter_available(base_url: &str, api_key: &str) -> bool {
if api_key.trim().is_empty() {
return false;
}
let Ok(client) = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(4))
.build()
else {
return false;
};
client
.get(format!("{}/models", base_url.trim_end_matches('/')))
.bearer_auth(api_key)
.send()
.map(|response| response.status().is_success())
.unwrap_or(false)
}
fn extract_chat_message_content(data: &Value) -> Option<String> {
let content = data
.get("choices")
.and_then(Value::as_array)
.and_then(|choices| choices.first())
.and_then(|choice| choice.get("message"))
.and_then(|message| message.get("content"))?;
match content {
Value::String(value) => Some(value.to_string()),
Value::Array(parts) => {
let text = parts
.iter()
.filter_map(|part| match part {
Value::String(value) => Some(value.as_str()),
Value::Object(object) => object
.get("text")
.or_else(|| object.get("content"))
.and_then(Value::as_str),
_ => None,
})
.collect::<Vec<_>>()
.join("");
if text.is_empty() { None } else { Some(text) }
}
_ => None,
}
}
fn truncate_error_body(body: &str) -> String {
const MAX_ERROR_BODY_CHARS: usize = 2000;
let mut truncated = body.chars().take(MAX_ERROR_BODY_CHARS).collect::<String>();
if body.chars().count() > MAX_ERROR_BODY_CHARS {
truncated.push_str("...");
}
truncated
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_comma_separated_fallback_models() {
assert_eq!(
parse_openrouter_fallback_models("a, b,,c "),
vec!["a".to_string(), "b".to_string(), "c".to_string()]
);
}
#[test]
fn extracts_text_from_openai_style_message() {
let data = json!({
"choices": [
{"message": {"content": [{"type": "text", "text": "Hej"}]}}
]
});
assert_eq!(extract_chat_message_content(&data), Some("Hej".to_string()));
}
}
+403
View File
@@ -0,0 +1,403 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::{Context, Result, bail};
use regex::Regex;
use crate::config::{
AUTO_I18N_COMMENT_PREFIX, AUTO_I18N_UNCHANGED_COMMENT, is_auto_i18n_unchanged_comment,
};
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct Entry {
pub comments: Vec<String>,
pub references: Vec<String>,
pub msgctxt: Option<String>,
pub msgid: String,
pub msgstr: String,
pub line_number: usize,
}
impl Entry {
pub fn with_msgid(msgid: impl Into<String>) -> Self {
Self {
msgid: msgid.into(),
..Self::default()
}
}
fn with_line_number(line_number: usize) -> Self {
Self {
line_number,
..Self::default()
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Translation {
pub msgid: String,
pub msgstr: String,
pub msgctxt: Option<String>,
pub reviewed_unchanged: bool,
pub notes: String,
}
impl Translation {
pub fn new(
msgctxt: Option<String>,
msgid: impl Into<String>,
msgstr: impl Into<String>,
) -> Self {
Self {
msgid: msgid.into(),
msgstr: msgstr.into(),
msgctxt,
reviewed_unchanged: false,
notes: String::new(),
}
}
}
pub fn parse_po(content: &str) -> Result<Vec<Entry>> {
let mut entries = Vec::new();
let normalized = content.replace("\r\n", "\n");
let mut current: Option<Entry> = None;
let mut current_field: Option<Field> = None;
let mut is_header = true;
for (index, line) in normalized.split('\n').enumerate() {
if line.starts_with("#. ") || line.starts_with("# ") {
current
.get_or_insert_with(|| Entry::with_line_number(index))
.comments
.push(line.to_string());
} else if line.starts_with("#: ") {
current
.get_or_insert_with(|| Entry::with_line_number(index))
.references
.push(line.to_string());
} else if let Some(token) = line.strip_prefix("msgctxt ") {
let entry = current.get_or_insert_with(|| Entry::with_line_number(index));
entry.msgctxt = Some(parse_po_token(token, index)?);
current_field = Some(Field::Msgctxt);
} else if let Some(token) = line.strip_prefix("msgid ") {
let entry = current.get_or_insert_with(|| Entry::with_line_number(index));
entry.msgid = parse_po_token(token, index)?;
current_field = Some(Field::Msgid);
} else if let Some(token) = line.strip_prefix("msgstr ") {
let entry = current.get_or_insert_with(|| Entry::with_line_number(index));
entry.msgstr = parse_po_token(token, index)?;
current_field = Some(Field::Msgstr);
} else if line.starts_with('"') && line.ends_with('"') {
if let (Some(entry), Some(field)) = (current.as_mut(), current_field) {
let value = parse_po_token(line, index)?;
match field {
Field::Msgctxt => {
if let Some(msgctxt) = entry.msgctxt.as_mut() {
msgctxt.push_str(&value);
}
}
Field::Msgid => entry.msgid.push_str(&value),
Field::Msgstr => entry.msgstr.push_str(&value),
}
}
} else if line.is_empty() && current.is_some() {
let entry = current.take().expect("checked current exists");
if is_header && entry.msgid.is_empty() {
is_header = false;
} else if !entry.msgid.is_empty() {
entries.push(entry);
}
current_field = None;
}
}
if let Some(entry) = current
&& !entry.msgid.is_empty()
{
entries.push(entry);
}
Ok(entries)
}
#[derive(Clone, Copy)]
enum Field {
Msgctxt,
Msgid,
Msgstr,
}
pub fn parse_po_token(token: &str, line_number: usize) -> Result<String> {
let value = serde_json::from_str::<serde_json::Value>(token)
.with_context(|| format!("Failed to parse PO string on line {}", line_number + 1))?;
match value {
serde_json::Value::String(value) => Ok(value),
_ => bail!("PO string on line {} is not a string", line_number + 1),
}
}
pub fn rebuild_po_allow_replacing(content: &str, translations: &[Translation]) -> Result<String> {
let translation_map = translations
.iter()
.map(|translation| {
Ok((
entry_key(translation.msgctxt.as_deref(), &translation.msgid)?,
translation,
))
})
.collect::<Result<Vec<_>>>()?;
let normalized = content.replace("\r\n", "\n");
let trimmed = normalized.trim_end();
let split_re = Regex::new(r"\n{2,}").expect("valid split regex");
let blocks = split_re.split(trimmed);
let mut rebuilt = Vec::new();
for block in blocks {
rebuilt.push(rebuild_block_allow_replacing(block, &translation_map)?);
}
Ok(format!("{}\n", rebuilt.join("\n\n")))
}
fn rebuild_block_allow_replacing(
block: &str,
translation_map: &[(String, &Translation)],
) -> Result<String> {
let lines = block
.split('\n')
.filter(|line| !is_auto_i18n_unchanged_comment(line))
.map(str::to_string)
.collect::<Vec<_>>();
let Some(msgid_range) = field_range(&lines, "msgid") else {
return Ok(block.to_string());
};
let Some(msgstr_range) = field_range(&lines, "msgstr") else {
return Ok(block.to_string());
};
let msgid = read_field_value(&lines, msgid_range)?;
let msgctxt = field_range(&lines, "msgctxt")
.map(|range| read_field_value(&lines, range))
.transpose()?;
let key = entry_key(msgctxt.as_deref(), &msgid)?;
let Some((_, translation)) = translation_map
.iter()
.find(|(candidate, _)| *candidate == key)
else {
return Ok(lines.join("\n"));
};
let mut next_lines = Vec::new();
if translation.reviewed_unchanged {
next_lines.push(AUTO_I18N_UNCHANGED_COMMENT.to_string());
}
next_lines.extend_from_slice(&lines[..msgstr_range.0]);
next_lines.push(format!("msgstr \"{}\"", escape_po(&translation.msgstr)));
next_lines.extend_from_slice(&lines[msgstr_range.1..]);
Ok(next_lines.join("\n"))
}
pub fn reset_po_translations(content: &str) -> Result<String> {
let translations = parse_po(content)?
.into_iter()
.map(|entry| Translation::new(entry.msgctxt, entry.msgid, ""))
.collect::<Vec<_>>();
rebuild_po_allow_replacing(content, &translations)
}
pub fn entry_key(msgctxt: Option<&str>, msgid: &str) -> Result<String> {
serde_json::to_string(&(msgctxt, msgid)).context("failed to serialize PO entry key")
}
#[derive(Clone, Copy)]
struct Range(usize, usize);
fn field_range(lines: &[String], field_name: &str) -> Option<Range> {
let start = lines
.iter()
.position(|line| line.starts_with(&format!("{field_name} ")))?;
let mut end = start + 1;
while end < lines.len() && lines[end].starts_with('"') && lines[end].ends_with('"') {
end += 1;
}
Some(Range(start, end))
}
fn read_field_value(lines: &[String], range: Range) -> Result<String> {
let Range(start, end) = range;
let mut value = parse_po_token(
lines[start]
.split_once(' ')
.map(|(_field, token)| token)
.unwrap_or_default(),
start,
)?;
for (index, line) in lines.iter().enumerate().take(end).skip(start + 1) {
value.push_str(&parse_po_token(line, index)?);
}
Ok(value)
}
pub fn escape_po(value: &str) -> String {
value
.replace('\\', "\\\\")
.replace('"', "\\\"")
.replace('\n', "\\n")
.replace('\t', "\\t")
}
pub fn extract_translator_comments(entry: &Entry) -> Vec<String> {
entry
.comments
.iter()
.filter_map(|line| line.strip_prefix("#. "))
.map(str::trim)
.filter(|comment| {
!comment.is_empty()
&& !is_auto_i18n_comment(comment)
&& !is_placeholder_comment(comment)
})
.map(str::to_string)
.collect()
}
pub fn extract_placeholder_hints(entry: &Entry) -> Vec<String> {
entry
.comments
.iter()
.filter_map(|line| line.strip_prefix("#. "))
.map(str::trim)
.filter(|comment| !comment.is_empty() && is_placeholder_comment(comment))
.map(str::to_string)
.collect()
}
pub fn is_auto_i18n_comment(comment: &str) -> bool {
comment.to_lowercase().starts_with(AUTO_I18N_COMMENT_PREFIX)
}
pub fn is_placeholder_comment(comment: &str) -> bool {
comment.to_lowercase().starts_with("placeholder {") && comment.contains("}:")
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_po() -> &'static str {
"msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain\\n\"\n\n\
#. Greeting shown on the welcome screen.\n\
#. placeholder {0}: user.name\n\
#: src/example.tsx:1\n\
msgctxt \"welcome title\"\n\
msgid \"Hello \\\"world\\\"\"\n\
msgstr \"\"\n\n\
#. auto-i18n: reviewed unchanged\n\
#: src/example.tsx:2\n\
msgctxt \"verb\"\n\
msgid \"Delete\"\n\
msgstr \"Delete\"\n\n\
#: src/example.tsx:3\n\
msgctxt \"keyboard key\"\n\
msgid \"Delete\"\n\
msgstr \"\"\n\n\
#: src/example.tsx:4\n\
msgid \"Line one\\n\"\n\
\"Line two\"\n\
msgstr \"\"\n"
}
#[test]
fn parses_context_comments_duplicates_and_multiline() {
let entries = parse_po(sample_po()).unwrap();
assert_eq!(entries.len(), 4);
let welcome = entries
.iter()
.find(|entry| entry.msgctxt.as_deref() == Some("welcome title"))
.unwrap();
assert_eq!(welcome.msgid, "Hello \"world\"");
assert_eq!(
extract_translator_comments(welcome),
vec!["Greeting shown on the welcome screen."]
);
assert_eq!(
extract_placeholder_hints(welcome),
vec!["placeholder {0}: user.name"]
);
let multiline = entries
.iter()
.find(|entry| entry.msgid.starts_with("Line"))
.unwrap();
assert_eq!(multiline.msgid, "Line one\nLine two");
}
#[test]
fn rebuilds_by_context_and_manages_unchanged_markers() {
let rebuilt = rebuild_po_allow_replacing(
sample_po(),
&[
Translation::new(
Some("welcome title".to_string()),
"Hello \"world\"",
"Bonjour \"monde\"",
),
Translation::new(Some("verb".to_string()), "Delete", "Supprimer"),
Translation {
msgctxt: Some("keyboard key".to_string()),
msgid: "Delete".to_string(),
msgstr: "Delete".to_string(),
reviewed_unchanged: true,
notes: String::new(),
},
],
)
.unwrap();
let entries = parse_po(&rebuilt).unwrap();
assert_eq!(
entries
.iter()
.find(|entry| entry.msgctxt.as_deref() == Some("welcome title"))
.unwrap()
.msgstr,
"Bonjour \"monde\""
);
assert_eq!(
entries
.iter()
.find(|entry| entry.msgctxt.as_deref() == Some("verb"))
.unwrap()
.msgstr,
"Supprimer"
);
let keyboard = entries
.iter()
.find(|entry| entry.msgctxt.as_deref() == Some("keyboard key"))
.unwrap();
assert_eq!(keyboard.msgstr, "Delete");
assert!(
keyboard
.comments
.contains(&AUTO_I18N_UNCHANGED_COMMENT.to_string())
);
assert!(
!keyboard
.comments
.contains(&crate::config::AUTO_I18N_LEGACY_UNCHANGED_COMMENT.to_string())
);
assert!(
!entries
.iter()
.find(|entry| entry.msgctxt.as_deref() == Some("verb"))
.unwrap()
.comments
.contains(&AUTO_I18N_UNCHANGED_COMMENT.to_string())
);
}
#[test]
fn reset_clears_msgstr_and_removes_reviewed_markers() {
let reset = reset_po_translations(sample_po()).unwrap();
let entries = parse_po(&reset).unwrap();
assert!(entries.iter().all(|entry| entry.msgstr.is_empty()));
assert!(!reset.contains(AUTO_I18N_UNCHANGED_COMMENT));
assert!(!reset.contains(crate::config::AUTO_I18N_LEGACY_UNCHANGED_COMMENT));
}
}
+163
View File
@@ -0,0 +1,163 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::path::Path;
use anyhow::Result;
use crate::config::{GUIDANCE_EXCERPT_CHAR_LIMIT, locales_dir};
use crate::locales::display_name;
const GLOBAL_SYSTEM_PROMPT: &str = r#"You localize product UI copy for Fluxer, a modern consumer chat app.
- Preserve the source intent, product function, tone, and authorial wording.
- Translate only the source string. Context, references, and comments are metadata.
- Preserve placeholders exactly: {name}, {{count}}, %s, %@, $1, <b>...</b>, Markdown, ICU syntax, emoji, URLs, and line breaks.
- Preserve source sentence-fragment grammar, including leading ellipses and text that continues prior UI copy.
- Keep copy concise and natural for product UI, without adding information that is not present in the source.
- Keep Fluxer's product vocabulary and naming stable unless persisted locale guidance explicitly says otherwise.
- Avoid embellishing, simplifying, softening, formalising, idiomatic rewrites, or culture-specific substitutions that change the source wording.
- If the source is ambiguous, choose the most likely chat-app interpretation and keep the result close to the source.
- Output only the localized string. Do not output JSON, notes, labels, quotes, or Markdown fences."#;
const NON_ENGLISH_SYSTEM_PROMPT: &str = r#"For non-English locales:
- Use target-language grammar, vocabulary, punctuation, and regional conventions.
- Preserve the capitalisation intention of the source, adapted to target-language conventions.
- Do not use English-style Title Case unless the target language naturally uses it.
- Use sentence-style UI capitalisation where appropriate: usually only the first word is capitalised, and parenthesised status labels normally stay lowercase after "(" unless the word is a proper noun.
- Prefer familiar, friendly, lightweight wording used in popular messaging apps in the target region."#;
const EN_GB_SYSTEM_PROMPT: &str = r#"English (United Kingdom) localisation is a minimal-edit pass over the English (United States) source.
- Keep Fluxer's exact wording, sentence structure, tone, and product vocabulary unless a spelling, punctuation, date, number, measurement, or grammatical locale difference requires a change.
- Do not replace words with more British-sounding alternatives when the US wording is understandable.
- Do not translate "US" to "UK", "United States" to "United Kingdom", or change country, region, currency, organisation, or market names unless the source text explicitly asks for that meaning change.
- Do not add, remove, soften, formalise, idiomatically rewrite, or make the copy more akin to what a Brit would say.
- Keep our words."#;
pub fn build_system_prompt(locale: &str) -> String {
[
Some(format!(
"You are a professional English (United States) (en-US) to {} ({locale}) translator.",
display_name(locale)
)),
Some(format!(
"Produce exactly one {} localization for Fluxer.",
display_name(locale)
)),
Some(GLOBAL_SYSTEM_PROMPT.to_string()),
build_locale_system_prompt(locale),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
.join("\n\n")
}
pub fn build_locale_system_prompt(locale: &str) -> Option<String> {
if locale == "en-GB" {
Some(EN_GB_SYSTEM_PROMPT.to_string())
} else if !locale.starts_with("en") {
Some(NON_ENGLISH_SYSTEM_PROMPT.to_string())
} else {
None
}
}
pub fn build_target_locale_prompt(locale: &str) -> String {
format!(
"Target locale: {}.\nUse the persisted locale guidance below as the source of truth for voice, tone, terminology, punctuation, and regional conventions.",
display_name(locale)
)
}
pub fn load_prompt_guidance(app_dir: &Path, locale: &str) -> Result<Vec<String>> {
Ok([load_guidance_file(
&format!("Locale guidance for {}", display_name(locale)),
&locales_dir(app_dir)
.join(locale)
.join("LOCALIZATION_PROMPT.md"),
)?]
.into_iter()
.flatten()
.collect())
}
fn load_guidance_file(label: &str, file_path: &Path) -> Result<Option<String>> {
if !file_path.exists() {
return Ok(None);
}
let excerpt = compact_guidance_excerpt(
&std::fs::read_to_string(file_path)?,
GUIDANCE_EXCERPT_CHAR_LIMIT,
);
Ok((!excerpt.is_empty()).then(|| format!("{label}:\n{excerpt}")))
}
pub fn compact_guidance_excerpt(content: &str, limit: usize) -> String {
let mut compact = content
.replace("\r\n", "\n")
.split('\n')
.map(str::trim)
.collect::<Vec<_>>()
.join("\n");
while compact.contains("\n\n\n") {
compact = compact.replace("\n\n\n", "\n\n");
}
let compact = compact.trim().to_string();
if compact.len() <= limit {
return compact;
}
let limit = floor_char_boundary(&compact, limit);
let truncated = &compact[..limit];
let last_break = [
truncated.rfind("\n\n"),
truncated.rfind('\n'),
truncated.rfind(". "),
]
.into_iter()
.flatten()
.max()
.unwrap_or(0);
let boundary = if last_break >= limit * 3 / 4 {
last_break
} else {
limit
};
format!("{}\n[excerpt truncated]", compact[..boundary].trim_end())
}
fn floor_char_boundary(value: &str, limit: usize) -> usize {
let mut boundary = limit.min(value.len());
while !value.is_char_boundary(boundary) {
boundary -= 1;
}
boundary
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn en_gb_prompt_keeps_strict_preservation_guidance() {
let prompt = build_system_prompt("en-GB");
assert!(prompt.contains("Do not translate \"US\" to \"UK\""));
assert!(prompt.contains("Keep our words."));
}
#[test]
fn compact_guidance_truncates_on_clean_boundary() {
let content = "First paragraph.\n\nSecond paragraph that should be truncated.\n\nThird.";
let compact = compact_guidance_excerpt(content, 35);
assert!(compact.ends_with("[excerpt truncated]"));
assert!(compact.starts_with("First paragraph."));
}
#[test]
fn compact_guidance_truncates_utf8_safely() {
let content = "Brand term: caf\u{00e9} caf\u{00e9} caf\u{00e9} caf\u{00e9}.";
let compact = compact_guidance_excerpt(content, 18);
assert!(compact.ends_with("[excerpt truncated]"));
}
}
+275
View File
@@ -0,0 +1,275 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, OpenOptions};
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
const REVIEWED_UNCHANGED_VERSION: u32 = 1;
const LOCK_TIMEOUT: Duration = Duration::from_secs(30);
const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(50);
#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
pub struct ReviewedUnchangedEntry {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub msgctxt: Option<String>,
pub msgid: String,
}
impl ReviewedUnchangedEntry {
pub fn new(msgctxt: Option<&str>, msgid: &str) -> Self {
Self {
msgctxt: msgctxt.map(str::to_string),
msgid: msgid.to_string(),
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
struct ReviewedUnchangedFile {
version: u32,
#[serde(default)]
locales: BTreeMap<String, Vec<ReviewedUnchangedEntry>>,
}
impl Default for ReviewedUnchangedFile {
fn default() -> Self {
Self {
version: REVIEWED_UNCHANGED_VERSION,
locales: BTreeMap::new(),
}
}
}
pub struct ReviewedUnchangedStore {
path: PathBuf,
data: ReviewedUnchangedFile,
dirty_locales: BTreeSet<String>,
}
impl ReviewedUnchangedStore {
pub fn load(path: impl Into<PathBuf>) -> Result<Self> {
let path = path.into();
let data = read_store_file(&path)?;
Ok(Self {
path,
data,
dirty_locales: BTreeSet::new(),
})
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn contains(&self, locale: &str, msgctxt: Option<&str>, msgid: &str) -> bool {
let needle = ReviewedUnchangedEntry::new(msgctxt, msgid);
self.data
.locales
.get(locale)
.is_some_and(|entries| entries.binary_search(&needle).is_ok())
}
pub fn mark(&mut self, locale: &str, msgctxt: Option<&str>, msgid: &str) {
let entry = ReviewedUnchangedEntry::new(msgctxt, msgid);
let entries = self.data.locales.entry(locale.to_string()).or_default();
match entries.binary_search(&entry) {
Ok(_) => {}
Err(index) => {
entries.insert(index, entry);
self.dirty_locales.insert(locale.to_string());
}
}
}
pub fn unmark(&mut self, locale: &str, msgctxt: Option<&str>, msgid: &str) {
let entry = ReviewedUnchangedEntry::new(msgctxt, msgid);
let Some(entries) = self.data.locales.get_mut(locale) else {
return;
};
let Ok(index) = entries.binary_search(&entry) else {
return;
};
entries.remove(index);
if entries.is_empty() {
self.data.locales.remove(locale);
}
self.dirty_locales.insert(locale.to_string());
}
pub fn clear_locale(&mut self, locale: &str) {
if self.data.locales.remove(locale).is_some() {
self.dirty_locales.insert(locale.to_string());
}
}
pub fn save_if_dirty(&mut self) -> Result<()> {
if self.dirty_locales.is_empty() {
return Ok(());
}
if let Some(parent) = self.path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
}
let _lock = SidecarLock::acquire(&self.path)?;
let mut merged = read_store_file(&self.path)?;
for locale in &self.dirty_locales {
match self.data.locales.get(locale) {
Some(entries) if !entries.is_empty() => {
let mut entries = entries.clone();
normalize_entries(&mut entries);
merged.locales.insert(locale.clone(), entries);
}
_ => {
merged.locales.remove(locale);
}
}
}
write_store_file(&self.path, &merged)?;
self.data = merged;
self.dirty_locales.clear();
Ok(())
}
}
fn read_store_file(path: &Path) -> Result<ReviewedUnchangedFile> {
let content = match fs::read_to_string(path) {
Ok(content) => content,
Err(error) if error.kind() == ErrorKind::NotFound => {
return Ok(ReviewedUnchangedFile::default());
}
Err(error) => {
return Err(error).with_context(|| format!("failed to read {}", path.display()));
}
};
let mut data = serde_json::from_str::<ReviewedUnchangedFile>(&content)
.with_context(|| format!("failed to parse {}", path.display()))?;
if data.version != REVIEWED_UNCHANGED_VERSION {
bail!(
"unsupported reviewed-unchanged sidecar version {} in {}",
data.version,
path.display()
);
}
for entries in data.locales.values_mut() {
normalize_entries(entries);
}
data.locales.retain(|_, entries| !entries.is_empty());
Ok(data)
}
fn normalize_entries(entries: &mut Vec<ReviewedUnchangedEntry>) {
entries.sort();
entries.dedup();
}
fn write_store_file(path: &Path, data: &ReviewedUnchangedFile) -> Result<()> {
let mut content = serde_json::to_string_pretty(data)
.with_context(|| format!("failed to serialize {}", path.display()))?;
content.push('\n');
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
let temp_path = path.with_file_name(format!(
".{}.{}.{}.tmp",
path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("auto-i18n-reviewed-unchanged.json"),
std::process::id(),
timestamp
));
fs::write(&temp_path, content)
.with_context(|| format!("failed to write {}", temp_path.display()))?;
fs::rename(&temp_path, path)
.with_context(|| format!("failed to replace {}", path.display()))?;
Ok(())
}
struct SidecarLock {
path: PathBuf,
}
impl SidecarLock {
fn acquire(sidecar_path: &Path) -> Result<Self> {
let lock_path = sidecar_path.with_extension("json.lock");
let started = Instant::now();
loop {
match OpenOptions::new()
.write(true)
.create_new(true)
.open(&lock_path)
{
Ok(_) => {
return Ok(Self { path: lock_path });
}
Err(error) if error.kind() == ErrorKind::AlreadyExists => {
if started.elapsed() >= LOCK_TIMEOUT {
bail!(
"timed out waiting for reviewed-unchanged sidecar lock {}",
lock_path.display()
);
}
thread::sleep(LOCK_POLL_INTERVAL);
}
Err(error) => {
return Err(error)
.with_context(|| format!("failed to create {}", lock_path.display()));
}
}
}
}
}
impl Drop for SidecarLock {
fn drop(&mut self) {
let _ = fs::remove_file(&self.path);
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn stores_entries_sorted_and_deduplicated() {
let temp = tempdir().unwrap();
let path = temp.path().join("reviewed.json");
let mut store = ReviewedUnchangedStore::load(&path).unwrap();
store.mark("de", Some("button"), "Save");
store.mark("de", None, "Audio");
store.mark("de", None, "Audio");
store.save_if_dirty().unwrap();
let saved = fs::read_to_string(&path).unwrap();
assert!(saved.contains("\"version\": 1"));
let loaded = ReviewedUnchangedStore::load(&path).unwrap();
assert!(loaded.contains("de", None, "Audio"));
assert!(loaded.contains("de", Some("button"), "Save"));
assert!(!loaded.contains("de", Some("button"), "Audio"));
}
#[test]
fn merge_save_preserves_other_locale_changes() {
let temp = tempdir().unwrap();
let path = temp.path().join("reviewed.json");
let mut first = ReviewedUnchangedStore::load(&path).unwrap();
let mut second = ReviewedUnchangedStore::load(&path).unwrap();
first.mark("de", None, "Audio");
second.mark("fr", None, "Avatar");
first.save_if_dirty().unwrap();
second.save_if_dirty().unwrap();
let loaded = ReviewedUnchangedStore::load(&path).unwrap();
assert!(loaded.contains("de", None, "Audio"));
assert!(loaded.contains("fr", None, "Avatar"));
}
}
File diff suppressed because it is too large Load Diff
+445
View File
@@ -0,0 +1,445 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::cmp::Reverse;
use anyhow::{Result, bail};
use regex::Regex;
use crate::po::Entry;
const EN_GB_PROTECTED_REGION_TERMS: &[&str] = &[
"United States of America",
"United States",
"U.S.",
"USA",
"US",
];
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct IcuControl {
pub argument: String,
pub kind: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TokenAlias {
pub token: String,
pub alias: String,
}
pub fn should_keep_unchanged(source: &str, locale: &str) -> bool {
if locale == "en-US" {
return false;
}
if source.trim().is_empty() {
return true;
}
if !has_any_letter(source) {
return true;
}
if is_preserved_token_composition(source) {
return true;
}
is_preserved_artifact(source)
}
pub fn has_any_letter(source: &str) -> bool {
source.chars().any(char::is_alphabetic)
}
pub fn is_preserved_artifact(source: &str) -> bool {
let trimmed = source.trim();
if [
r"(?i)^https?://\S+$",
r"(?i)^(?:mailto:|tel:|file:|app://|fluxer://)\S+$",
r"^[@#][\w-]+$",
r"^\{[^{}]+\}$",
r"^<[^>]+>$",
r"^%[@sd]$",
r"^\$\d+$",
r#"^[\s{}#,._:/+()\[\]'"-]+$"#,
r"^[A-Z][A-Z0-9_./:+-]+$",
]
.iter()
.any(|pattern| {
Regex::new(pattern)
.expect("valid artifact regex")
.is_match(trimmed)
}) {
return true;
}
Regex::new(r"^[\w./:+-]+$")
.expect("valid word artifact regex")
.is_match(trimmed)
&& Regex::new(r"[\d_./:+-]")
.expect("valid word artifact marker regex")
.is_match(trimmed)
}
pub fn is_preserved_token_composition(source: &str) -> bool {
let entry = Entry {
msgid: source.to_string(),
..Entry::default()
};
let mut tokens = extract_preserved_tokens(&entry);
if tokens.is_empty() {
return false;
}
tokens.sort_by_key(|token| Reverse(token.len()));
let mut remainder = source.to_string();
for token in tokens {
remainder = Regex::new(&regex::escape(&token))
.expect("valid preserved token cleanup regex")
.replace_all(&remainder, "")
.into_owned();
}
remainder
.chars()
.all(|character| character.is_whitespace() || is_preserved_token_separator(character))
}
fn is_preserved_token_separator(character: char) -> bool {
matches!(
character,
':' | ','
| '.'
| '/'
| '\\'
| '-'
| ''
| '—'
| '('
| ')'
| '['
| ']'
| '{'
| '}'
| '|'
| '·'
| '•'
| '#'
| '+'
| '\''
| '"'
)
}
pub fn validate_localization(entry: &Entry, localized: &str) -> Result<()> {
if !entry.msgid.trim().is_empty() && localized.trim().is_empty() {
bail!("Localized string is empty");
}
let source_tokens = extract_preserved_tokens(entry);
let localized_entry = Entry {
msgid: localized.to_string(),
comments: entry.comments.clone(),
..Entry::default()
};
let localized_tokens = extract_preserved_tokens(&localized_entry);
for token in source_tokens {
if !localized_tokens.contains(&token) {
bail!("Localized string did not preserve token {token}");
}
}
for control in extract_icu_controls(&entry.msgid) {
let pattern = Regex::new(&format!(
r"\{{\s*{}\s*,\s*{}\s*,",
regex::escape(&control.argument),
regex::escape(&control.kind)
))
.expect("valid ICU validation regex");
if !pattern.is_match(localized) {
bail!(
"Localized string did not preserve ICU {} argument {}",
control.kind,
control.argument
);
}
}
Ok(())
}
pub fn validate_locale_specific_localization(
entry: &Entry,
locale: &str,
localized: &str,
) -> Result<()> {
if locale != "en-GB" {
return Ok(());
}
for term in extract_en_gb_protected_terms(&entry.msgid) {
if !contains_exact_term(localized, term) {
bail!("en-GB localization changed protected source term {term}");
}
}
Ok(())
}
pub fn extract_en_gb_protected_terms(source: &str) -> Vec<&'static str> {
EN_GB_PROTECTED_REGION_TERMS
.iter()
.copied()
.filter(|term| contains_exact_term(source, term))
.collect()
}
pub fn contains_exact_term(source: &str, term: &str) -> bool {
source.match_indices(term).any(|(start, _match)| {
let before_ok = source[..start]
.chars()
.next_back()
.is_none_or(|before| !is_word_char(before));
let end = start + term.len();
let after_ok = source[end..]
.chars()
.next()
.is_none_or(|after| !is_word_char(after));
before_ok && after_ok
})
}
fn is_word_char(value: char) -> bool {
value == '_' || value.is_alphanumeric()
}
pub fn normalize_localized_capitalization(entry: &Entry, locale: &str, localized: &str) -> String {
if locale.starts_with("en") || !entry.msgid.starts_with('(') || !localized.starts_with('(') {
return localized.to_string();
}
let mut chars = localized.chars();
let Some(first) = chars.next() else {
return localized.to_string();
};
let Some(second) = chars.next() else {
return localized.to_string();
};
let second_string = second.to_string();
if second_string != second.to_uppercase().to_string() {
return localized.to_string();
}
let rest = chars.as_str();
format!("{first}{}{rest}", second.to_lowercase())
}
pub fn extract_preserved_tokens(entry: &Entry) -> Vec<String> {
let mut tokens = Vec::new();
for token in extract_top_level_brace_tokens(&entry.msgid) {
push_unique(&mut tokens, token);
}
let token_re = Regex::new(r"%[@sd]|\$\d+|<[^>]+>|https?://\S+|:[a-zA-Z0-9_+-]+:")
.expect("valid preserved token regex");
for matched in token_re.find_iter(&entry.msgid) {
push_unique(&mut tokens, matched.as_str().to_string());
}
tokens
}
pub fn extract_top_level_brace_tokens(source: &str) -> Vec<String> {
let mut tokens = Vec::new();
let mut depth = 0usize;
let mut start: Option<usize> = None;
let double_brace_re = Regex::new(r"^\{\{[^{}]+\}\}$").expect("valid double brace regex");
let brace_re = Regex::new(r"^\{[^{},]+\}$").expect("valid brace regex");
for (index, character) in source.char_indices() {
if character == '{' {
if depth == 0 {
start = Some(index);
}
depth += 1;
} else if character == '}' && depth > 0 {
depth -= 1;
if depth == 0 {
if let Some(start_index) = start {
let token = &source[start_index..index + character.len_utf8()];
if double_brace_re.is_match(token) || brace_re.is_match(token) {
tokens.push(token.to_string());
}
}
start = None;
}
}
}
tokens
}
pub fn has_icu_control(source: &str) -> bool {
Regex::new(r"\{[^{}]+,\s*(?:plural|select|selectordinal)\s*,")
.expect("valid ICU detection regex")
.is_match(source)
}
pub fn extract_icu_controls(source: &str) -> Vec<IcuControl> {
Regex::new(r"\{\s*([^{}\s,]+)\s*,\s*(plural|select|selectordinal)\s*,")
.expect("valid ICU control regex")
.captures_iter(source)
.map(|captures| IcuControl {
argument: captures[1].to_string(),
kind: captures[2].to_string(),
})
.collect()
}
pub fn build_masked_source(entry: &Entry) -> (String, Vec<TokenAlias>) {
let token_aliases = extract_preserved_tokens(entry)
.into_iter()
.enumerate()
.map(|(index, token)| TokenAlias {
token,
alias: format!("{{FLUXER_TOKEN_{index}}}"),
})
.collect::<Vec<_>>();
(
mask_text_with_aliases(&entry.msgid, &token_aliases),
token_aliases,
)
}
pub fn mask_text_with_aliases(text: &str, token_aliases: &[TokenAlias]) -> String {
let mut masked = text.to_string();
let mut aliases = token_aliases.iter().collect::<Vec<_>>();
aliases.sort_by_key(|alias| std::cmp::Reverse(alias.token.len()));
for item in aliases {
masked = Regex::new(&regex::escape(&item.token))
.expect("valid token mask regex")
.replace_all(&masked, item.alias.as_str())
.into_owned();
}
masked
}
pub fn restore_masked_tokens(localized: &str, token_aliases: &[TokenAlias]) -> String {
let mut restored = localized.to_string();
for item in token_aliases {
restored = restored.replace(&item.alias, &item.token);
}
restored
}
pub fn build_token_alias_context(token_aliases: &[TokenAlias]) -> String {
let mut lines = vec!["Literal protected aliases used in the source string:".to_string()];
lines.extend(
token_aliases
.iter()
.map(|item| format!("{} = {}", item.alias, item.token)),
);
lines.extend([
"Aliases are required substrings, not words or labels to localize.".to_string(),
"Copy every alias exactly in the localized output, with the same spelling and braces.".to_string(),
"Do not replace an alias with translated words, even if the original token name looks meaningful."
.to_string(),
"Translate only the human-readable words around aliases.".to_string(),
]);
lines.join("\n")
}
fn push_unique(tokens: &mut Vec<String>, token: String) {
if !tokens.contains(&token) {
tokens.push(token);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_sources_that_should_stay_unchanged() {
assert!(should_keep_unchanged("", "de"));
assert!(should_keep_unchanged("https://fluxer.app/docs", "de"));
assert!(should_keep_unchanged("{productName}", "de"));
assert!(should_keep_unchanged("{authorName} {description}", "de"));
assert!(should_keep_unchanged(
"{emojiName}: {reactionCountText}, {actionText}",
"de"
));
assert!(should_keep_unchanged("{start} {end}", "de"));
assert!(should_keep_unchanged("C++", "de"));
assert!(!should_keep_unchanged("NCMEC {ncmecReportId}", "de"));
assert!(!should_keep_unchanged("Hello", "de"));
assert!(!should_keep_unchanged("{productName}", "en-US"));
}
#[test]
fn extracts_masks_and_restores_preserved_tokens() {
let entry = Entry::with_msgid("{productName} needs <0>{permission}</0> %@ $1 :wave:");
assert_eq!(
extract_preserved_tokens(&entry),
vec![
"{productName}",
"{permission}",
"<0>",
"</0>",
"%@",
"$1",
":wave:"
]
);
let (masked, aliases) = build_masked_source(&entry);
assert!(masked.contains("{FLUXER_TOKEN_0}"));
assert_eq!(
restore_masked_tokens(&masked, &aliases),
"{productName} needs <0>{permission}</0> %@ $1 :wave:"
);
}
#[test]
fn validates_tokens_and_icu_controls() {
let entry = Entry {
msgid: "{0} {1, plural, one {member} other {members}}".to_string(),
comments: vec!["#. placeholder {0}: count".to_string()],
..Entry::default()
};
validate_localization(&entry, "{0} {1, plural, one {miembro} other {miembros}}").unwrap();
let error =
validate_localization(&entry, "{0} {2, plural, one {x} other {x}}").unwrap_err();
assert!(
error
.to_string()
.contains("did not preserve ICU plural argument 1")
);
let missing_token =
validate_localization(&Entry::with_msgid("Open {productName}"), "Open").unwrap_err();
assert!(missing_token.to_string().contains("{productName}"));
}
#[test]
fn validates_en_gb_protected_region_terms() {
let entry = Entry::with_msgid("US color settings for the United States");
validate_locale_specific_localization(
&entry,
"en-GB",
"US colour settings for the United States",
)
.unwrap();
let error = validate_locale_specific_localization(
&entry,
"en-GB",
"UK colour settings for the United Kingdom",
)
.unwrap_err();
assert!(
error
.to_string()
.contains("protected source term United States")
);
}
#[test]
fn normalizes_non_english_parenthesized_capitalization() {
assert_eq!(
normalize_localized_capitalization(
&Entry::with_msgid("(No content)"),
"es-ES",
"(Sin contenido)",
),
"(sin contenido)"
);
assert_eq!(
normalize_localized_capitalization(
&Entry::with_msgid("(No content)"),
"en-GB",
"(No content)",
),
"(No content)"
);
}
}
+867
View File
@@ -0,0 +1,867 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use std::collections::HashMap;
use std::ops::Range;
use std::path::PathBuf;
use anyhow::{Context, Result, bail};
use crate::po::{Entry, Translation};
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum StaticTsCatalogKind {
SimpleMessages,
EmailTemplates,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StaticTsCatalogConfig {
pub kind: StaticTsCatalogKind,
pub source_path: PathBuf,
pub source_export: String,
pub locale_function: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ObjectRange {
body: Range<usize>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ParsedStringProperty {
key: String,
value: String,
value_range: Range<usize>,
line_number: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ParsedObjectProperty {
key: String,
body: Range<usize>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct StaticSourceMessage {
context: String,
source: String,
line_number: usize,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct StaticTargetCatalog {
object_body: Range<usize>,
values: HashMap<String, ParsedStringProperty>,
template_objects: HashMap<String, ParsedObjectProperty>,
}
pub fn read_static_ts_entries(
config: &StaticTsCatalogConfig,
source_content: &str,
target_content: &str,
reset: bool,
) -> Result<Vec<Entry>> {
let source_messages = parse_source_messages(config, source_content)?;
let target_catalog = parse_target_catalog(config, target_content)?;
Ok(source_messages
.into_iter()
.map(|source| {
let msgstr = if reset {
String::new()
} else {
target_catalog
.values
.get(&source.context)
.map(|target| target.value.clone())
.unwrap_or_default()
};
Entry {
comments: static_comments(config, &source.context),
references: vec![format!(
"#: {}:{}",
config.source_path.display(),
source.line_number
)],
msgctxt: Some(source.context),
msgid: source.source,
msgstr,
line_number: source.line_number,
}
})
.collect())
}
pub fn rebuild_static_ts_allow_replacing(
config: &StaticTsCatalogConfig,
source_content: &str,
target_content: &str,
translations: &[Translation],
) -> Result<String> {
let source_messages = parse_source_messages(config, source_content)?;
let target_catalog = parse_target_catalog(config, target_content)?;
let translations_by_context = translations
.iter()
.filter_map(|translation| {
translation
.msgctxt
.as_ref()
.map(|context| (context.clone(), translation))
})
.collect::<HashMap<_, _>>();
let mut replacements = Vec::new();
for (context, translation) in &translations_by_context {
if let Some(target) = target_catalog.values.get(context) {
replacements.push((
target.value_range.clone(),
escape_ts_single_quoted(&translation.msgstr),
));
}
}
match config.kind {
StaticTsCatalogKind::SimpleMessages => {
add_missing_simple_translations(
target_content,
&target_catalog,
&source_messages,
&translations_by_context,
&mut replacements,
);
}
StaticTsCatalogKind::EmailTemplates => {
add_missing_email_translations(
target_content,
&target_catalog,
&source_messages,
&translations_by_context,
&mut replacements,
)?;
}
}
apply_replacements(target_content, replacements)
}
pub fn reset_static_ts_translations(
config: &StaticTsCatalogConfig,
target_content: &str,
) -> Result<String> {
let target_catalog = parse_target_catalog(config, target_content)?;
let replacements = target_catalog
.values
.values()
.map(|target| (target.value_range.clone(), "''".to_string()))
.collect::<Vec<_>>();
apply_replacements(target_content, replacements)
}
fn static_comments(config: &StaticTsCatalogConfig, context: &str) -> Vec<String> {
match config.kind {
StaticTsCatalogKind::SimpleMessages => {
vec![format!("#. Static catalog key: {context}")]
}
StaticTsCatalogKind::EmailTemplates => {
vec![format!("#. Email template field: {context}")]
}
}
}
fn parse_source_messages(
config: &StaticTsCatalogConfig,
source_content: &str,
) -> Result<Vec<StaticSourceMessage>> {
let object = find_const_object(source_content, &config.source_export)?;
match config.kind {
StaticTsCatalogKind::SimpleMessages => {
parse_string_properties(source_content, &object.body).map(|properties| {
properties
.into_iter()
.map(|property| StaticSourceMessage {
context: property.key,
source: property.value,
line_number: property.line_number,
})
.collect()
})
}
StaticTsCatalogKind::EmailTemplates => {
let mut messages = Vec::new();
for template in parse_object_properties(source_content, &object.body)? {
for field in parse_string_properties(source_content, &template.body)? {
if field.key == "subject" || field.key == "body" {
messages.push(StaticSourceMessage {
context: format!("{}.{}", template.key, field.key),
source: field.value,
line_number: field.line_number,
});
}
}
}
Ok(messages)
}
}
}
fn parse_target_catalog(
config: &StaticTsCatalogConfig,
target_content: &str,
) -> Result<StaticTargetCatalog> {
let object = find_call_object(target_content, &config.locale_function)?;
match config.kind {
StaticTsCatalogKind::SimpleMessages => {
let values = parse_string_properties(target_content, &object.body)?
.into_iter()
.map(|property| (property.key.clone(), property))
.collect::<HashMap<_, _>>();
Ok(StaticTargetCatalog {
object_body: object.body,
values,
template_objects: HashMap::new(),
})
}
StaticTsCatalogKind::EmailTemplates => {
let mut values = HashMap::new();
let mut template_objects = HashMap::new();
for template in parse_object_properties(target_content, &object.body)? {
for field in parse_string_properties(target_content, &template.body)? {
if field.key == "subject" || field.key == "body" {
values.insert(format!("{}.{}", template.key, field.key), field);
}
}
template_objects.insert(template.key.clone(), template);
}
Ok(StaticTargetCatalog {
object_body: object.body,
values,
template_objects,
})
}
}
}
fn add_missing_simple_translations(
target_content: &str,
target_catalog: &StaticTargetCatalog,
source_messages: &[StaticSourceMessage],
translations_by_context: &HashMap<String, &Translation>,
replacements: &mut Vec<(Range<usize>, String)>,
) {
let lines = source_messages
.iter()
.filter(|source| !target_catalog.values.contains_key(&source.context))
.filter_map(|source| {
translations_by_context
.get(&source.context)
.map(|translation| {
format!(
"\t{}: {},",
escape_property_key(&source.context),
escape_ts_single_quoted(&translation.msgstr)
)
})
})
.collect::<Vec<_>>();
if !lines.is_empty() {
replacements.push(append_to_object_body(
target_content,
&target_catalog.object_body,
lines,
));
}
}
fn add_missing_email_translations(
target_content: &str,
target_catalog: &StaticTargetCatalog,
source_messages: &[StaticSourceMessage],
translations_by_context: &HashMap<String, &Translation>,
replacements: &mut Vec<(Range<usize>, String)>,
) -> Result<()> {
let source_by_context = source_messages
.iter()
.map(|source| (source.context.clone(), source.source.clone()))
.collect::<HashMap<_, _>>();
let mut source_template_order = Vec::new();
for source in source_messages {
let (template_key, _field) = split_email_context(&source.context)?;
if !source_template_order
.iter()
.any(|candidate| candidate == template_key)
{
source_template_order.push(template_key.to_string());
}
}
for template_key in &source_template_order {
if let Some(template) = target_catalog.template_objects.get(template_key) {
let lines = ["subject", "body"]
.into_iter()
.filter_map(|field| {
let context = format!("{template_key}.{field}");
if target_catalog.values.contains_key(&context) {
return None;
}
translations_by_context.get(&context).map(|translation| {
format!(
"\t\t{field}: {},",
escape_ts_single_quoted(&translation.msgstr)
)
})
})
.collect::<Vec<_>>();
if !lines.is_empty() {
replacements.push(append_to_object_body(target_content, &template.body, lines));
}
}
}
let mut template_blocks = Vec::new();
for template_key in &source_template_order {
if target_catalog.template_objects.contains_key(template_key) {
continue;
}
let subject_context = format!("{template_key}.subject");
let body_context = format!("{template_key}.body");
let has_selected_translation = translations_by_context.contains_key(&subject_context)
|| translations_by_context.contains_key(&body_context);
if !has_selected_translation {
continue;
}
let subject = translations_by_context
.get(&subject_context)
.map(|translation| translation.msgstr.as_str())
.or_else(|| source_by_context.get(&subject_context).map(String::as_str))
.with_context(|| format!("missing source subject for email template {template_key}"))?;
let body = translations_by_context
.get(&body_context)
.map(|translation| translation.msgstr.as_str())
.or_else(|| source_by_context.get(&body_context).map(String::as_str))
.with_context(|| format!("missing source body for email template {template_key}"))?;
template_blocks.push(format!(
"\t{}: {{\n\t\tsubject: {},\n\t\tbody: {},\n\t}},",
escape_identifier_or_key(template_key),
escape_ts_single_quoted(subject),
escape_ts_single_quoted(body)
));
}
if !template_blocks.is_empty() {
replacements.push(append_to_object_body(
target_content,
&target_catalog.object_body,
template_blocks,
));
}
Ok(())
}
fn split_email_context(context: &str) -> Result<(&str, &str)> {
let Some((template, field)) = context.rsplit_once('.') else {
bail!("email catalog context must be <template>.<subject|body>: {context}");
};
if field != "subject" && field != "body" {
bail!("email catalog field must be subject or body: {context}");
}
Ok((template, field))
}
fn append_to_object_body(
content: &str,
body: &Range<usize>,
lines: Vec<String>,
) -> (Range<usize>, String) {
let body_content = &content[body.clone()];
let trimmed_len = body_content.trim_end().len();
let insert_at = body.start + trimmed_len;
let has_existing_entries = !body_content[..trimmed_len].trim().is_empty();
let mut insertion = String::new();
if has_existing_entries && !body_content[..trimmed_len].trim_end().ends_with(',') {
insertion.push(',');
}
insertion.push('\n');
insertion.push_str(&lines.join("\n"));
(insert_at..insert_at, insertion)
}
fn apply_replacements(
content: &str,
mut replacements: Vec<(Range<usize>, String)>,
) -> Result<String> {
replacements.sort_by(|(left, _), (right, _)| {
right
.start
.cmp(&left.start)
.then_with(|| right.end.cmp(&left.end))
});
let mut rebuilt = content.to_string();
let mut previous_start = content.len() + 1;
for (range, replacement) in replacements {
if range.end > previous_start {
bail!("overlapping static TS catalog replacements");
}
rebuilt.replace_range(range.clone(), &replacement);
previous_start = range.start;
}
Ok(rebuilt)
}
fn find_const_object(content: &str, export_name: &str) -> Result<ObjectRange> {
let export_index = content
.find(export_name)
.with_context(|| format!("failed to find static catalog export {export_name}"))?;
let open_relative = content[export_index..]
.find('{')
.with_context(|| format!("failed to find object literal for {export_name}"))?;
object_range_at(content, export_index + open_relative)
}
fn find_call_object(content: &str, function_name: &str) -> Result<ObjectRange> {
let mut search_start = 0;
while let Some(relative_index) = content[search_start..].find(function_name) {
let function_index = search_start + relative_index;
let mut index = function_index + function_name.len();
index = skip_ws_comments(content, index, content.len())?;
if byte_at(content, index) != Some(b'(') {
search_start = index;
continue;
}
index = skip_ws_comments(content, index + 1, content.len())?;
if byte_at(content, index) != Some(b'{') {
search_start = index;
continue;
}
return object_range_at(content, index);
}
bail!("failed to find static locale function call {function_name}");
}
fn object_range_at(content: &str, open_index: usize) -> Result<ObjectRange> {
if byte_at(content, open_index) != Some(b'{') {
bail!("expected object literal at byte {open_index}");
}
let close_index = find_matching_brace(content, open_index)?;
Ok(ObjectRange {
body: open_index + 1..close_index,
})
}
fn find_matching_brace(content: &str, open_index: usize) -> Result<usize> {
let bytes = content.as_bytes();
let mut depth = 0usize;
let mut index = open_index;
let mut state = ScanState::Normal;
while index < bytes.len() {
match state {
ScanState::Normal => match bytes[index] {
b'\'' | b'"' | b'`' => {
state = ScanState::String {
quote: bytes[index],
escaped: false,
};
index += 1;
}
b'/' if byte_at(content, index + 1) == Some(b'/') => {
state = ScanState::LineComment;
index += 2;
}
b'/' if byte_at(content, index + 1) == Some(b'*') => {
state = ScanState::BlockComment;
index += 2;
}
b'{' => {
depth += 1;
index += 1;
}
b'}' => {
depth = depth.saturating_sub(1);
if depth == 0 {
return Ok(index);
}
index += 1;
}
_ => index += 1,
},
ScanState::String { quote, escaped } => {
if escaped {
state = ScanState::String {
quote,
escaped: false,
};
} else if bytes[index] == b'\\' {
state = ScanState::String {
quote,
escaped: true,
};
} else if bytes[index] == quote {
state = ScanState::Normal;
}
index += 1;
}
ScanState::LineComment => {
if bytes[index] == b'\n' {
state = ScanState::Normal;
}
index += 1;
}
ScanState::BlockComment => {
if bytes[index] == b'*' && byte_at(content, index + 1) == Some(b'/') {
state = ScanState::Normal;
index += 2;
} else {
index += 1;
}
}
}
}
bail!("unterminated object literal")
}
#[derive(Clone, Copy)]
enum ScanState {
Normal,
String { quote: u8, escaped: bool },
LineComment,
BlockComment,
}
fn parse_string_properties(
content: &str,
body: &Range<usize>,
) -> Result<Vec<ParsedStringProperty>> {
let mut properties = Vec::new();
let mut index = body.start;
while index < body.end {
index = skip_ws_comments(content, index, body.end)?;
if index >= body.end {
break;
}
if byte_at(content, index) == Some(b',') {
index += 1;
continue;
}
let (key, next_index) = parse_property_key(content, index, body.end)?;
index = skip_ws_comments(content, next_index, body.end)?;
if byte_at(content, index) != Some(b':') {
bail!("expected ':' after property key {key}");
}
index = skip_ws_comments(content, index + 1, body.end)?;
let value = parse_ts_string(content, index, body.end)
.with_context(|| format!("expected string literal value for property {key}"))?;
index = value.value_range.end;
properties.push(ParsedStringProperty {
key,
value: value.value,
value_range: value.value_range,
line_number: line_number_at(content, index),
});
}
Ok(properties)
}
fn parse_object_properties(
content: &str,
body: &Range<usize>,
) -> Result<Vec<ParsedObjectProperty>> {
let mut properties = Vec::new();
let mut index = body.start;
while index < body.end {
index = skip_ws_comments(content, index, body.end)?;
if index >= body.end {
break;
}
if byte_at(content, index) == Some(b',') {
index += 1;
continue;
}
let (key, next_index) = parse_property_key(content, index, body.end)?;
index = skip_ws_comments(content, next_index, body.end)?;
if byte_at(content, index) != Some(b':') {
bail!("expected ':' after object property key {key}");
}
index = skip_ws_comments(content, index + 1, body.end)?;
let object = object_range_at(content, index)
.with_context(|| format!("expected object literal value for property {key}"))?;
index = object.body.end + 1;
properties.push(ParsedObjectProperty {
key,
body: object.body,
});
}
Ok(properties)
}
fn parse_property_key(content: &str, index: usize, end: usize) -> Result<(String, usize)> {
match byte_at(content, index) {
Some(b'\'') | Some(b'"') => {
let parsed = parse_ts_string(content, index, end)?;
Ok((parsed.value, parsed.value_range.end))
}
Some(byte) if is_identifier_start(byte) => {
let mut next = index + 1;
while next < end && byte_at(content, next).is_some_and(is_identifier_part) {
next += 1;
}
Ok((content[index..next].to_string(), next))
}
_ => bail!("expected property key at byte {index}"),
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ParsedTsString {
value: String,
value_range: Range<usize>,
}
fn parse_ts_string(content: &str, start: usize, end: usize) -> Result<ParsedTsString> {
let quote = byte_at(content, start).context("expected TS string literal")?;
if quote != b'\'' && quote != b'"' {
bail!("expected TS string literal at byte {start}");
}
let mut value = String::new();
let mut index = start + 1;
while index < end {
let character = content[index..]
.chars()
.next()
.context("invalid TS string literal")?;
if character as u32 == quote as u32 {
return Ok(ParsedTsString {
value,
value_range: start..index + 1,
});
}
if character == '\\' {
index += 1;
let escaped = content[index..]
.chars()
.next()
.context("unterminated escape sequence in TS string literal")?;
match escaped {
'\'' => value.push('\''),
'"' => value.push('"'),
'\\' => value.push('\\'),
'n' => value.push('\n'),
'r' => value.push('\r'),
't' => value.push('\t'),
'b' => value.push('\u{0008}'),
'f' => value.push('\u{000c}'),
'v' => value.push('\u{000b}'),
'0' => value.push('\0'),
'x' => {
let (parsed, next_index) = parse_hex_escape(content, index + 1, 2)?;
value.push(parsed);
index = next_index;
continue;
}
'u' => {
let (parsed, next_index) = parse_unicode_escape(content, index + 1)?;
value.push(parsed);
index = next_index;
continue;
}
'\n' => {}
'\r' => {}
other => value.push(other),
}
index += escaped.len_utf8();
} else {
value.push(character);
index += character.len_utf8();
}
}
bail!("unterminated TS string literal at byte {start}")
}
fn parse_hex_escape(content: &str, start: usize, len: usize) -> Result<(char, usize)> {
let end = start + len;
let value = u32::from_str_radix(
content
.get(start..end)
.with_context(|| format!("invalid hex escape at byte {start}"))?,
16,
)
.with_context(|| format!("invalid hex escape at byte {start}"))?;
let character =
char::from_u32(value).with_context(|| format!("invalid code point at byte {start}"))?;
Ok((character, end))
}
fn parse_unicode_escape(content: &str, start: usize) -> Result<(char, usize)> {
if byte_at(content, start) == Some(b'{') {
let close_relative = content[start + 1..]
.find('}')
.with_context(|| format!("unterminated unicode escape at byte {start}"))?;
let digits_start = start + 1;
let digits_end = digits_start + close_relative;
let value = u32::from_str_radix(&content[digits_start..digits_end], 16)
.with_context(|| format!("invalid unicode escape at byte {start}"))?;
let character =
char::from_u32(value).with_context(|| format!("invalid code point at byte {start}"))?;
Ok((character, digits_end + 1))
} else {
parse_hex_escape(content, start, 4)
}
}
fn skip_ws_comments(content: &str, mut index: usize, end: usize) -> Result<usize> {
while index < end {
match byte_at(content, index) {
Some(byte) if byte.is_ascii_whitespace() => index += 1,
Some(b'/') if byte_at(content, index + 1) == Some(b'/') => {
index += 2;
while index < end && byte_at(content, index) != Some(b'\n') {
index += 1;
}
}
Some(b'/') if byte_at(content, index + 1) == Some(b'*') => {
let close_relative = content[index + 2..end]
.find("*/")
.with_context(|| format!("unterminated block comment at byte {index}"))?;
index = index + 2 + close_relative + 2;
}
_ => break,
}
}
Ok(index)
}
fn byte_at(content: &str, index: usize) -> Option<u8> {
content.as_bytes().get(index).copied()
}
fn line_number_at(content: &str, index: usize) -> usize {
content[..index]
.bytes()
.filter(|byte| *byte == b'\n')
.count()
+ 1
}
fn is_identifier_start(byte: u8) -> bool {
byte.is_ascii_alphabetic() || byte == b'_' || byte == b'$'
}
fn is_identifier_part(byte: u8) -> bool {
is_identifier_start(byte) || byte.is_ascii_digit()
}
fn is_identifier(value: &str) -> bool {
let mut bytes = value.bytes();
let Some(first) = bytes.next() else {
return false;
};
is_identifier_start(first) && bytes.all(is_identifier_part)
}
fn escape_property_key(value: &str) -> String {
escape_ts_single_quoted(value)
}
fn escape_identifier_or_key(value: &str) -> String {
if is_identifier(value) {
value.to_string()
} else {
escape_ts_single_quoted(value)
}
}
fn escape_ts_single_quoted(value: &str) -> String {
let mut escaped = String::from("'");
for character in value.chars() {
match character {
'\\' => escaped.push_str("\\\\"),
'\'' => escaped.push_str("\\'"),
'\n' => escaped.push_str("\\n"),
'\r' => escaped.push_str("\\r"),
'\t' => escaped.push_str("\\t"),
'\u{2028}' => escaped.push_str("\\u2028"),
'\u{2029}' => escaped.push_str("\\u2029"),
other => escaped.push(other),
}
}
escaped.push('\'');
escaped
}
#[cfg(test)]
mod tests {
use super::*;
fn simple_config() -> StaticTsCatalogConfig {
StaticTsCatalogConfig {
kind: StaticTsCatalogKind::SimpleMessages,
source_path: PathBuf::from("SourceMessages.ts"),
source_export: "SOURCE_MESSAGES".to_string(),
locale_function: "defineLocaleMessages".to_string(),
}
}
fn email_config() -> StaticTsCatalogConfig {
StaticTsCatalogConfig {
kind: StaticTsCatalogKind::EmailTemplates,
source_path: PathBuf::from("EmailMessages.ts"),
source_export: "EMAIL_MESSAGES".to_string(),
locale_function: "defineEmailMessages".to_string(),
}
}
#[test]
fn reads_and_updates_simple_static_catalogs() {
let source = "export const SOURCE_MESSAGES = {\n\t'hello.world': 'Hello world',\n\t'bye': 'Bye',\n} as const;\n";
let target = "import {defineLocaleMessages} from '../Messages';\n\nexport const DE = defineLocaleMessages({\n\t'hello.world': 'Hallo Welt',\n});\n";
let entries = read_static_ts_entries(&simple_config(), source, target, false).unwrap();
assert_eq!(entries.len(), 2);
assert_eq!(entries[0].msgctxt.as_deref(), Some("hello.world"));
assert_eq!(entries[0].msgstr, "Hallo Welt");
assert_eq!(entries[1].msgstr, "");
let rebuilt = rebuild_static_ts_allow_replacing(
&simple_config(),
source,
target,
&[
Translation::new(Some("hello.world".to_string()), "Hello world", "Hallo"),
Translation::new(Some("bye".to_string()), "Bye", "Tschüss"),
],
)
.unwrap();
assert!(rebuilt.contains("'hello.world': 'Hallo',"));
assert!(rebuilt.contains("'bye': 'Tschüss',"));
}
#[test]
fn reads_and_updates_email_template_catalogs() {
let source = "export const EMAIL_MESSAGES = {\n\twelcome: {\n\t\tsubject: 'Welcome to {product_name}',\n\t\tbody: 'Hello {username}\\nWelcome.',\n\t},\n\treset_password: {\n\t\tsubject: 'Reset password',\n\t\tbody: 'Use {resetUrl}',\n\t},\n} as const;\n";
let target = "import {defineEmailMessages} from '../EmailMessages';\n\nexport const DE = defineEmailMessages({\n\twelcome: {\n\t\tsubject: 'Willkommen bei {product_name}',\n\t\tbody: 'Hallo {username}\\nWillkommen.',\n\t},\n});\n";
let entries = read_static_ts_entries(&email_config(), source, target, false).unwrap();
assert_eq!(entries.len(), 4);
assert_eq!(entries[0].msgctxt.as_deref(), Some("welcome.subject"));
assert_eq!(entries[1].msgid, "Hello {username}\nWelcome.");
assert_eq!(entries[2].msgstr, "");
let rebuilt = rebuild_static_ts_allow_replacing(
&email_config(),
source,
target,
&[
Translation::new(
Some("welcome.body".to_string()),
"Hello {username}\nWelcome.",
"Hallo {username}\nGuten Tag.",
),
Translation::new(
Some("reset_password.subject".to_string()),
"Reset password",
"Passwort zurücksetzen",
),
],
)
.unwrap();
assert!(rebuilt.contains("body: 'Hallo {username}\\nGuten Tag.',"));
assert!(rebuilt.contains("reset_password: {"));
assert!(rebuilt.contains("subject: 'Passwort zurücksetzen',"));
assert!(rebuilt.contains("body: 'Use {resetUrl}',"));
}
#[test]
fn reset_clears_existing_static_catalog_values() {
let target =
"export const DE = defineLocaleMessages({\n\t'hello.world': 'Hallo Welt',\n});\n";
let reset = reset_static_ts_translations(&simple_config(), target).unwrap();
assert!(reset.contains("'hello.world': '',"));
}
}
@@ -0,0 +1,12 @@
{
"schema": 1,
"scenario": "event_log_replay_determinism",
"seed": 4242,
"workload": "network-partition-during-screen-share",
"snapshot_hash": "f957df06",
"event_count": 22,
"final_tick": 401,
"description": "FNV-1a hash of the simulator snapshot after the network-partition-during-screen-share scenario at seed 4242. The hash must be byte-identical across macOS, Linux, Windows. Any divergence indicates a platform-leak: floating-point determinism, HashMap iteration order, or non-deterministic clock/random.",
"recorded_on_platform": "macos",
"recorded_at_iso8601": "2026-06-09T00:00:00Z"
}
+103
View File
@@ -0,0 +1,103 @@
schema_version = 1
[platforms.macos]
description = "macOS ARM64 (this host) - real ScreenCaptureKit, CoreAudio, Metal via wgpu"
runner = "local"
cargo_build = true
ts_driver = true
scenarios = [
"scenario_audio_capture_to_mix",
"scenario_screen_capture_to_pool",
"scenario_gpu_device_loss_recovery",
"scenario_event_log_replay_determinism",
"scenario_ffi_airlock_negative_space",
"scenario_encoder_handoff_dryrun",
]
[platforms.linux-fedora]
description = "Fedora 42 ARM64 in Parallels VM, reached via prlctl - real PipeWire, Vulkan via wgpu"
runner = "prlctl"
prlctl_vm = "Fedora 42 ARM64"
prlctl_source = "/media/psf/fluxer-dev"
cargo_build = true
ts_driver = true
scenarios = [
"scenario_audio_capture_to_mix",
"scenario_screen_capture_to_pool",
"scenario_gpu_device_loss_recovery",
"scenario_event_log_replay_determinism",
"scenario_ffi_airlock_negative_space",
"scenario_encoder_handoff_dryrun",
]
[platforms.windows-tailnet]
description = "Windows 11 desktop on Tailnet (desktop-5ba12hj) - NVIDIA MX450 + Intel Iris Xe, WGC, DXGI, D3D11, NVENC, QSV"
runner = "ssh"
ssh_host = "desktop-5ba12hj"
ssh_user = "Hampus"
cargo_build = true
ts_driver = true
scenarios = [
"scenario_audio_capture_to_mix",
"scenario_screen_capture_to_pool",
"scenario_gpu_device_loss_recovery",
"scenario_event_log_replay_determinism",
"scenario_ffi_airlock_negative_space",
"scenario_encoder_handoff_dryrun",
]
[platforms.windows-vm]
description = "Windows 11 ARM Parallels VM - ARM-only, limited GPU access"
runner = "prlctl"
prlctl_vm = "Windows 11"
cargo_build = true
ts_driver = true
scenarios = [
"scenario_audio_capture_to_mix",
"scenario_screen_capture_to_pool",
"scenario_event_log_replay_determinism",
"scenario_ffi_airlock_negative_space",
"scenario_encoder_handoff_dryrun",
]
[scenario.scenario_audio_capture_to_mix]
description = "Real PCM capture -> SourceRing -> AudioMixSession -> APM -> mix output"
expected_runtime_seconds = 5
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
driver = "rust"
subcommand = "audio-capture-to-mix"
[scenario.scenario_screen_capture_to_pool]
description = "Real screen capture -> FramePool with allocation-free steady state"
expected_runtime_seconds = 5
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
driver = "rust"
subcommand = "screen-capture-to-pool"
[scenario.scenario_gpu_device_loss_recovery]
description = "wgpu device loss -> GpuLossRegistry rebuild walk -> Nv12Packer ready"
expected_runtime_seconds = 10
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet"]
driver = "rust"
subcommand = "gpu-device-loss-recovery"
[scenario.scenario_event_log_replay_determinism]
description = "Fixed-seed simulator scenario - hash agreement across platforms"
expected_runtime_seconds = 20
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
driver = "typescript"
ts_entry = "tools/integration/ts-driver/event_log_replay_determinism.ts"
[scenario.scenario_ffi_airlock_negative_space]
description = "Malformed audio frame -> Rust receive-side assert panic with expected message"
expected_runtime_seconds = 3
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
driver = "rust"
subcommand = "ffi-airlock-negative"
[scenario.scenario_encoder_handoff_dryrun]
description = "EncoderInputRing - D3D11KeyedMutexBackend on Windows, CpuMemcpyBackend elsewhere - 8 slots, DTS offset, skip-don't-block"
expected_runtime_seconds = 5
applicable_platforms = ["macos", "linux-fedora", "windows-tailnet", "windows-vm"]
driver = "rust"
subcommand = "encoder-handoff-dryrun"
@@ -0,0 +1,3 @@
target/
target-*/
Cargo.lock
@@ -0,0 +1,25 @@
[package]
name = "fluxer_integration_driver"
version = "0.0.0"
edition = "2024"
license = "AGPL-3.0-or-later"
publish = false
[workspace]
resolver = "2"
[[bin]]
name = "integration-driver"
path = "src/main.rs"
[dependencies]
fluxer_screen_frame_bus = {path = "../../../fluxer_desktop/native/screen-frame-bus", features = ["wgpu"]}
fluxer_gpu_rebuild = {path = "../../../fluxer_desktop/native/gpu-rebuild", features = ["wgpu"]}
fluxer_audio_mix = {path = "../../../fluxer_desktop/native/audio-mix"}
fluxer_audio_apm = {path = "../../../fluxer_desktop/native/audio-apm"}
fluxer_nv12_gpu_pack = {path = "../../../fluxer_desktop/native/nv12-gpu-pack"}
fluxer_encoder_ring = {path = "../../../fluxer_desktop/native/encoder-ring"}
fluxer_rt_thread = {path = "../../../fluxer_desktop/native/rt-thread"}
wgpu = "29"
serde = {version = "1", features = ["derive"]}
serde_json = "1"
@@ -0,0 +1,50 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
#![deny(warnings)]
#![deny(clippy::unwrap_used)]
#![deny(clippy::panic)]
#![deny(clippy::too_many_lines)]
mod scenarios;
use std::process::ExitCode;
const USAGE: &str = "usage: integration-driver <scenario> [args...]\n\
scenarios:\n \
audio-capture-to-mix\n \
screen-capture-to-pool\n \
gpu-device-loss-recovery\n \
encoder-handoff-dryrun\n \
ffi-airlock-negative [invalid-sample-rate|valid-baseline]\n";
fn main() -> ExitCode {
let args: Vec<String> = std::env::args().collect();
assert!(!args.is_empty(), "argv must contain program name");
if args.len() < 2 {
eprintln!("{}", USAGE);
return ExitCode::from(64);
}
let scenario = args[1].as_str();
let tail: Vec<&str> = args[2..].iter().map(String::as_str).collect();
let result = match scenario {
"audio-capture-to-mix" => scenarios::audio_capture_to_mix::run(&tail),
"screen-capture-to-pool" => scenarios::screen_capture_to_pool::run(&tail),
"gpu-device-loss-recovery" => scenarios::gpu_device_loss_recovery::run(&tail),
"encoder-handoff-dryrun" => scenarios::encoder_handoff_dryrun::run(&tail),
"ffi-airlock-negative" => scenarios::ffi_airlock_negative::run(&tail),
other => {
eprintln!("unknown scenario {other}\n{USAGE}");
return ExitCode::from(64);
}
};
match result {
Ok(report) => {
println!("{}", report);
ExitCode::SUCCESS
}
Err(report) => {
println!("{}", report);
ExitCode::from(1)
}
}
}
@@ -0,0 +1,166 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use fluxer_audio_apm::{ApmConfigBuilder, AudioProcessor, StubAudioProcessor};
use fluxer_audio_mix::{AUDIO_OUTPUT_FRAMES, AudioMixSession, SourceRing};
use fluxer_rt_thread::TickInfo;
use serde_json::json;
use super::ScenarioReport;
const TONE_AMPLITUDE: i16 = 8_000;
const TONE_SAMPLE_RATE_HZ: u32 = 48_000;
const TONE_FREQUENCY_HZ: u32 = 440;
const TONE_TICK_COUNT: u64 = 4;
const APM_SAMPLE_RATE_HZ: u32 = 48_000;
const APM_CHANNELS: u16 = 1;
const APM_FRAME_SAMPLES: usize = 480;
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
assert_eq!(APM_FRAME_SAMPLES, 480, "APM 10 ms @ 48 kHz invariant");
let measurements = match drive_pipeline() {
Ok(m) => m,
Err(reason) => {
return Err(ScenarioReport::fail(
"audio_capture_to_mix",
json!({"reason": reason}),
Vec::new(),
));
}
};
let mut assertions = Vec::new();
if measurements.tone_mix_peak < (TONE_AMPLITUDE as i64) / 4 {
return Err(ScenarioReport::fail(
"audio_capture_to_mix",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec!["tone peak below quarter-amplitude floor".to_string()],
));
}
assertions.push("tone present in mix output above quarter-amplitude floor".to_string());
if measurements.apm_capture_frames_processed == 0 {
return Err(ScenarioReport::fail(
"audio_capture_to_mix",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec!["APM processed zero capture frames".to_string()],
));
}
assertions.push("APM processed at least one capture frame".to_string());
if measurements.mix_ticks_completed != TONE_TICK_COUNT {
return Err(ScenarioReport::fail(
"audio_capture_to_mix",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec![format!(
"expected {} mix ticks, observed {}",
TONE_TICK_COUNT, measurements.mix_ticks_completed
)],
));
}
assertions.push("mix session completed the full tick schedule".to_string());
Ok(ScenarioReport::pass(
"audio_capture_to_mix",
serde_json::to_value(&measurements).unwrap_or(json!({})),
assertions,
))
}
#[derive(Debug, serde::Serialize)]
struct Measurements {
tone_mix_peak: i64,
tone_mix_rms: i64,
mix_ticks_completed: u64,
apm_capture_frames_processed: u64,
source_ring_pushed_total: u64,
source_ring_drained_total: u64,
sample_rate_hz: u32,
tone_frequency_hz: u32,
}
fn drive_pipeline() -> Result<Measurements, String> {
let (mut producer, consumer) = SourceRing::create(8192, TONE_SAMPLE_RATE_HZ)
.map_err(|err| format!("ring create failed: {err:?}"))?;
let pushed_per_tick = AUDIO_OUTPUT_FRAMES;
let mut phase: u32 = 0;
let mut apm_buffer: Vec<i16> = vec![0i16; APM_FRAME_SAMPLES];
let apm_config = ApmConfigBuilder::new()
.aec(false)
.ns(false)
.agc(false)
.build();
let mut apm = StubAudioProcessor::new(apm_config, APM_SAMPLE_RATE_HZ, APM_CHANNELS)
.map_err(|err| format!("apm init failed: {err:?}"))?;
let mut source_ring_pushed_total: u64 = 0;
for _ in 0..TONE_TICK_COUNT {
let tone = generate_tone_chunk(pushed_per_tick, &mut phase);
assert_eq!(tone.len(), pushed_per_tick, "tone chunk length");
let mut apm_offset: usize = 0;
while apm_offset + APM_FRAME_SAMPLES <= tone.len() {
apm_buffer.copy_from_slice(&tone[apm_offset..apm_offset + APM_FRAME_SAMPLES]);
let _ = apm
.process_capture_frame(&mut apm_buffer, APM_SAMPLE_RATE_HZ, APM_CHANNELS)
.map_err(|err| format!("apm process failed: {err:?}"))?;
apm_offset += APM_FRAME_SAMPLES;
}
let pushed = producer.try_push_slice(&tone);
assert_eq!(pushed, tone.len(), "ring must accept full tone chunk");
source_ring_pushed_total = source_ring_pushed_total.saturating_add(pushed as u64);
}
let mut session = AudioMixSession::new(vec![consumer], AUDIO_OUTPUT_FRAMES)
.map_err(|err| format!("mix init failed: {err:?}"))?;
let mut peak_observed: i64 = 0;
let mut accumulator_sq: u64 = 0;
let mut counted: u64 = 0;
for tick_index in 0..TONE_TICK_COUNT {
let tick = synthetic_tick(tick_index);
let _ = session.tick(tick);
let output = session.last_output();
for sample in output.iter() {
let abs = (*sample as i64).abs();
if abs > peak_observed {
peak_observed = abs;
}
let sq = (*sample as i64).saturating_mul(*sample as i64);
accumulator_sq = accumulator_sq.saturating_add(sq as u64);
counted = counted.saturating_add(1);
}
}
let rms = accumulator_sq.checked_div(counted).unwrap_or(0).isqrt() as i64;
Ok(Measurements {
tone_mix_peak: peak_observed,
tone_mix_rms: rms,
mix_ticks_completed: session.ticks_completed(),
apm_capture_frames_processed: apm.capture_frames_processed(),
source_ring_pushed_total,
source_ring_drained_total: 0,
sample_rate_hz: TONE_SAMPLE_RATE_HZ,
tone_frequency_hz: TONE_FREQUENCY_HZ,
})
}
fn generate_tone_chunk(samples: usize, phase: &mut u32) -> Vec<i16> {
assert!(samples > 0, "tone chunk samples positive");
assert!(samples <= 1 << 14, "tone chunk within sanity cap");
let mut out = Vec::with_capacity(samples);
let period_samples = TONE_SAMPLE_RATE_HZ / TONE_FREQUENCY_HZ;
assert!(period_samples > 0, "period must be positive");
for _ in 0..samples {
let progress = (*phase as f32) / (period_samples as f32);
let radians = progress * std::f32::consts::TAU;
let value = (radians.sin() * TONE_AMPLITUDE as f32) as i16;
out.push(value);
*phase = phase.wrapping_add(1);
if *phase >= period_samples {
*phase = 0;
}
}
assert_eq!(out.len(), samples, "tone chunk length post-condition");
out
}
fn synthetic_tick(index: u64) -> TickInfo {
let scheduled_ns = index.saturating_mul(21_333_333);
TickInfo {
tick_index: index,
scheduled_ns,
actual_ns: scheduled_ns,
lag_ns: 0,
}
}
@@ -0,0 +1,166 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use fluxer_encoder_ring::{
CpuMemcpyBackend, EncoderInputRing, RingError, TextureFormat, apply_dts_offset,
compute_dts_offset_us,
};
use serde_json::json;
use super::ScenarioReport;
const FRAME_WIDTH_PX: u32 = 1280;
const FRAME_HEIGHT_PX: u32 = 720;
const RING_DEPTH_TARGET: u32 = 8;
const LAG_OVERSUBSCRIBE_COUNT: u32 = 4;
const FRAME_INTERVAL_US: u64 = 16_666;
const NUM_B_FRAMES: u32 = 2;
const FIRST_PTS_US: u64 = 0;
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
let measurements = match drive_ring() {
Ok(m) => m,
Err(reason) => {
return Err(ScenarioReport::fail(
"encoder_handoff_dryrun",
json!({"reason": reason}),
Vec::new(),
));
}
};
let mut assertions = Vec::new();
if measurements.completed_count != RING_DEPTH_TARGET as u64 {
return Err(ScenarioReport::fail(
"encoder_handoff_dryrun",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec![format!(
"expected {} completed slots, observed {}",
RING_DEPTH_TARGET, measurements.completed_count
)],
));
}
assertions.push("eight slots submitted and completed".to_string());
if measurements.dropped_count != LAG_OVERSUBSCRIBE_COUNT as u64 {
return Err(ScenarioReport::fail(
"encoder_handoff_dryrun",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec![format!(
"expected {} dropped under oversubscription, observed {}",
LAG_OVERSUBSCRIBE_COUNT, measurements.dropped_count
)],
));
}
assertions.push("oversubscribed submits report skip-don't-block drop counter".to_string());
if measurements.dispatched_count != RING_DEPTH_TARGET as u64 {
return Err(ScenarioReport::fail(
"encoder_handoff_dryrun",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec![format!(
"expected {} dispatched ready frames, observed {}",
RING_DEPTH_TARGET, measurements.dispatched_count
)],
));
}
assertions.push("dispatched_count equals submitted_count in steady state".to_string());
if measurements.dts_first_frame > measurements.pts_first_frame {
return Err(ScenarioReport::fail(
"encoder_handoff_dryrun",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec!["first DTS must not exceed PTS".to_string()],
));
}
assertions.push("DTS offset is non-positive given B-frames".to_string());
Ok(ScenarioReport::pass(
"encoder_handoff_dryrun",
serde_json::to_value(&measurements).unwrap_or(json!({})),
assertions,
))
}
#[derive(Debug, serde::Serialize)]
struct Measurements {
backend: &'static str,
submitted_count: u64,
completed_count: u64,
dispatched_count: u64,
dropped_count: u64,
capacity: usize,
pts_first_frame: u64,
dts_first_frame: u64,
dts_offset_us: i64,
texture_skipped_frames: u32,
}
fn drive_ring() -> Result<Measurements, String> {
let backend = CpuMemcpyBackend::new();
let mut ring: EncoderInputRing<CpuMemcpyBackend> = EncoderInputRing::new(backend);
ring.initialise(FRAME_WIDTH_PX, FRAME_HEIGHT_PX, TextureFormat::Nv12)
.map_err(|err| format!("ring init failed: {err:?}"))?;
assert_eq!(
ring.capacity(),
RING_DEPTH_TARGET as usize,
"ring capacity invariant"
);
assert_eq!(
ring.free_count(),
RING_DEPTH_TARGET as usize,
"fresh ring fully free"
);
for _ in 0..RING_DEPTH_TARGET {
ring.submit(noop_fill)
.map_err(|err| format!("first round submit failed: {err:?}"))?;
}
let mut texture_skipped_frames: u32 = 0;
for _ in 0..LAG_OVERSUBSCRIBE_COUNT {
match ring.submit(noop_fill) {
Err(RingError::FullDropped { .. }) => {
texture_skipped_frames = texture_skipped_frames.saturating_add(1);
}
Err(err) => return Err(format!("unexpected submit error: {err:?}")),
Ok(()) => return Err("oversubscribed submit must drop".to_string()),
}
}
let dts_offset_us = compute_dts_offset_us(FIRST_PTS_US, NUM_B_FRAMES, FRAME_INTERVAL_US);
assert!(dts_offset_us <= 0, "dts offset non-positive");
let pts_first_frame = FRAME_INTERVAL_US.saturating_mul(NUM_B_FRAMES as u64);
let dts_first_frame = apply_dts_offset(pts_first_frame, dts_offset_us);
let mut released_sequences: Vec<u64> = Vec::with_capacity(RING_DEPTH_TARGET as usize);
while let Some(ready) = ring.poll_next_ready() {
released_sequences.push(ready.sequence);
ring.release_completed(ready)
.map_err(|err| format!("release failed: {err:?}"))?;
}
assert_eq!(
released_sequences.len(),
RING_DEPTH_TARGET as usize,
"expected all submitted frames to drain"
);
for (idx, seq) in released_sequences.iter().enumerate() {
let expected = (idx + 1) as u64;
assert_eq!(*seq, expected, "FIFO release order");
}
let metrics = ring.metrics();
Ok(Measurements {
backend: backend_label(),
submitted_count: metrics.submitted_count,
completed_count: metrics.completed_count,
dispatched_count: metrics.dispatched_count,
dropped_count: metrics.dropped_count,
capacity: ring.capacity(),
pts_first_frame,
dts_first_frame,
dts_offset_us,
texture_skipped_frames,
})
}
#[cfg(target_os = "windows")]
const fn backend_label() -> &'static str {
"cpu_memcpy_with_d3d11_keyed_mutex_available"
}
#[cfg(not(target_os = "windows"))]
const fn backend_label() -> &'static str {
"cpu_memcpy_backend"
}
fn noop_fill(_slot: &mut fluxer_encoder_ring::CpuSlotHandle) {}
@@ -0,0 +1,234 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use serde_json::json;
use std::fmt;
use std::panic::{AssertUnwindSafe, catch_unwind};
use super::ScenarioReport;
const VALID_SAMPLE_RATE_HZ: u32 = 48_000;
const INVALID_SAMPLE_RATE_HZ: u32 = 44_100;
const VALID_CHANNELS: u32 = 1;
const VALID_FRAME_BYTES: u32 = 960;
const VALID_TIMESTAMP_NS: u64 = 1_000_000;
const AUDIO_FRAME_BYTES_MAX: u32 = 1 << 20;
const AUDIO_SAMPLE_RATES_HZ: [u32; 3] = [16_000, 32_000, 48_000];
const AUDIO_CHANNELS_VALID: [u32; 2] = [1, 2];
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct VoiceEngineV2AudioFrameInvariants {
sample_rate_hz: u32,
num_channels: u32,
frame_bytes: u32,
timestamp_ns: u64,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum VoiceEngineV2AssertError {
FrameBytesOutOfRange { bytes: u32, max: u32 },
SampleRateInvalid { hz: u32 },
ChannelsInvalid { channels: u32 },
TimestampRegressed { previous_ns: u64, received_ns: u64 },
}
impl fmt::Display for VoiceEngineV2AssertError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::FrameBytesOutOfRange { bytes, max } => {
write!(
formatter,
"AudioFrameBytesOutOfRange: audio frame bytes {bytes} not in (0, {max}]"
)
}
Self::SampleRateInvalid { hz } => {
write!(
formatter,
"AudioSampleRateInvalid: audio sample rate {hz} not in [16000, 32000, 48000]"
)
}
Self::ChannelsInvalid { channels } => {
write!(
formatter,
"AudioChannelsInvalid: audio channels {channels} not in [1, 2]"
)
}
Self::TimestampRegressed {
previous_ns,
received_ns,
} => {
write!(
formatter,
"AudioTimestampRegressed: audio timestamp {received_ns} did not exceed previous {previous_ns}"
)
}
}
}
}
fn check_audio_frame_invariants(
frame: VoiceEngineV2AudioFrameInvariants,
previous_timestamp_ns: Option<u64>,
) -> Result<(), VoiceEngineV2AssertError> {
if frame.frame_bytes == 0 || frame.frame_bytes > AUDIO_FRAME_BYTES_MAX {
return Err(VoiceEngineV2AssertError::FrameBytesOutOfRange {
bytes: frame.frame_bytes,
max: AUDIO_FRAME_BYTES_MAX,
});
}
if !AUDIO_SAMPLE_RATES_HZ.contains(&frame.sample_rate_hz) {
return Err(VoiceEngineV2AssertError::SampleRateInvalid {
hz: frame.sample_rate_hz,
});
}
if !AUDIO_CHANNELS_VALID.contains(&frame.num_channels) {
return Err(VoiceEngineV2AssertError::ChannelsInvalid {
channels: frame.num_channels,
});
}
if let Some(previous_ns) = previous_timestamp_ns
&& frame.timestamp_ns <= previous_ns
{
return Err(VoiceEngineV2AssertError::TimestampRegressed {
previous_ns,
received_ns: frame.timestamp_ns,
});
}
Ok(())
}
#[allow(clippy::panic)]
fn assert_audio_frame_invariants(
frame: VoiceEngineV2AudioFrameInvariants,
previous_timestamp_ns: Option<u64>,
) {
let result = check_audio_frame_invariants(frame, previous_timestamp_ns);
if let Err(err) = result {
panic!("{err}");
}
}
pub fn run(args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
let prior_hook = std::panic::take_hook();
std::panic::set_hook(Box::new(|_info| {}));
let result = run_inner(args);
std::panic::set_hook(prior_hook);
result
}
fn run_inner(args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
let mode = args.first().copied().unwrap_or("invalid-sample-rate");
let measurements = match mode {
"invalid-sample-rate" => exercise_invalid_sample_rate(),
"valid-baseline" => exercise_valid_baseline(),
other => {
return Err(ScenarioReport::fail(
"ffi_airlock_negative",
json!({"reason": format!("unknown mode: {other}")}),
Vec::new(),
));
}
};
match measurements {
Ok(m) => Ok(ScenarioReport::pass(
"ffi_airlock_negative",
serde_json::to_value(&m).unwrap_or(json!({})),
m.assertions,
)),
Err(report) => Err(report),
}
}
#[derive(Debug, serde::Serialize)]
struct Measurements {
mode: &'static str,
panic_observed: bool,
panic_message: Option<String>,
valid_baseline_passed: bool,
#[serde(skip_serializing)]
assertions: Vec<String>,
}
fn exercise_invalid_sample_rate() -> Result<Measurements, ScenarioReport> {
let frame = VoiceEngineV2AudioFrameInvariants {
sample_rate_hz: INVALID_SAMPLE_RATE_HZ,
num_channels: VALID_CHANNELS,
frame_bytes: VALID_FRAME_BYTES,
timestamp_ns: VALID_TIMESTAMP_NS,
};
let outcome = catch_unwind(AssertUnwindSafe(|| {
assert_audio_frame_invariants(frame, None);
}));
match outcome {
Ok(()) => Err(ScenarioReport::fail(
"ffi_airlock_negative",
json!({"mode": "invalid-sample-rate", "panic_observed": false}),
vec!["receive-side assertion failed to panic on invalid sample rate".to_string()],
)),
Err(payload) => {
let message = panic_payload_to_string(&payload);
if !message.contains("AudioSampleRateInvalid")
&& !message.contains("not in [16000, 32000, 48000]")
{
return Err(ScenarioReport::fail(
"ffi_airlock_negative",
json!({"mode": "invalid-sample-rate", "panic_observed": true, "panic_message": message}),
vec!["panic message did not mention AudioSampleRateInvalid".to_string()],
));
}
let assertions = vec![
"receive-side assertion panicked as expected".to_string(),
"panic message names AudioSampleRateInvalid".to_string(),
];
Ok(Measurements {
mode: "invalid-sample-rate",
panic_observed: true,
panic_message: Some(message),
valid_baseline_passed: false,
assertions,
})
}
}
}
fn exercise_valid_baseline() -> Result<Measurements, ScenarioReport> {
let frame = VoiceEngineV2AudioFrameInvariants {
sample_rate_hz: VALID_SAMPLE_RATE_HZ,
num_channels: VALID_CHANNELS,
frame_bytes: VALID_FRAME_BYTES,
timestamp_ns: VALID_TIMESTAMP_NS,
};
let outcome = catch_unwind(AssertUnwindSafe(|| {
assert_audio_frame_invariants(frame, None);
}));
match outcome {
Ok(()) => {
let assertions = vec!["valid-frame baseline did not panic".to_string()];
Ok(Measurements {
mode: "valid-baseline",
panic_observed: false,
panic_message: None,
valid_baseline_passed: true,
assertions,
})
}
Err(payload) => Err(ScenarioReport::fail(
"ffi_airlock_negative",
json!({
"mode": "valid-baseline",
"panic_observed": true,
"panic_message": panic_payload_to_string(&payload),
}),
vec!["valid baseline frame must not panic".to_string()],
)),
}
}
fn panic_payload_to_string(payload: &Box<dyn std::any::Any + Send>) -> String {
if let Some(s) = payload.downcast_ref::<&'static str>() {
return (*s).to_string();
}
if let Some(s) = payload.downcast_ref::<String>() {
return s.clone();
}
"<non-string panic payload>".to_string()
}
@@ -0,0 +1,163 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use fluxer_gpu_rebuild::{GpuLossCallback, GpuLossRegistry};
use fluxer_nv12_gpu_pack::Nv12Packer;
use fluxer_screen_frame_bus::gpu_loss::{
WgpuStagingBackend, WgpuStagingConfig, try_acquire_device,
};
use serde_json::json;
use super::ScenarioReport;
const PACKER_MAX_WIDTH: u32 = 1280;
const PACKER_MAX_HEIGHT: u32 = 720;
const STAGING_BYTE_LEN: u64 = 64 * 1024;
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
let (device, queue, _instance) = acquire_gpu_context()?;
let measurements = drive_rebuild(&device, &queue).map_err(|reason| {
ScenarioReport::fail(
"gpu_device_loss_recovery",
json!({"reason": reason}),
Vec::new(),
)
})?;
let assertions = check_measurements(&measurements)?;
Ok(ScenarioReport::pass(
"gpu_device_loss_recovery",
serde_json::to_value(&measurements).unwrap_or(json!({})),
assertions,
))
}
fn acquire_gpu_context() -> Result<(wgpu::Device, wgpu::Queue, wgpu::Instance), ScenarioReport> {
let acquired = std::panic::catch_unwind(std::panic::AssertUnwindSafe(try_acquire_device));
match acquired {
Ok(Some(triple)) => Ok(triple),
Ok(None) => Err(ScenarioReport::fail(
"gpu_device_loss_recovery",
json!({"reason": "no wgpu adapter available"}),
Vec::new(),
)),
Err(_) => Err(ScenarioReport::fail(
"gpu_device_loss_recovery",
json!({"reason": "wgpu adapter acquisition panicked"}),
Vec::new(),
)),
}
}
fn check_measurements(measurements: &Measurements) -> Result<Vec<String>, ScenarioReport> {
if measurements.released_count != 2 {
return Err(fail_with(
measurements,
format!(
"expected 2 release calls, observed {}",
measurements.released_count
),
));
}
if measurements.rebuilt_count != 2 {
return Err(fail_with(
measurements,
format!(
"expected 2 rebuilt owners, observed {}",
measurements.rebuilt_count
),
));
}
if measurements.failed_count != 0 {
return Err(fail_with(
measurements,
format!(
"expected 0 failures, observed {}",
measurements.failed_count
),
));
}
if !measurements.packer_ready_after_rebuild {
return Err(fail_with(
measurements,
"Nv12Packer not ready after rebuild".to_string(),
));
}
if !measurements.staging_ready_after_rebuild {
return Err(fail_with(
measurements,
"WgpuStagingBackend not ready after rebuild".to_string(),
));
}
Ok(vec![
"registry released both owners".to_string(),
"registry rebuilt both owners".to_string(),
"zero failures during rebuild walk".to_string(),
"Nv12Packer reports is_ready true after rebuild".to_string(),
"WgpuStagingBackend reports is_ready true after rebuild".to_string(),
])
}
fn fail_with(measurements: &Measurements, message: String) -> ScenarioReport {
ScenarioReport::fail(
"gpu_device_loss_recovery",
serde_json::to_value(measurements).unwrap_or(json!({})),
vec![message],
)
}
#[derive(Debug, serde::Serialize)]
struct Measurements {
released_count: u32,
rebuilt_count: u32,
failed_count: u32,
vacant_count: u32,
packer_ready_after_rebuild: bool,
staging_ready_after_rebuild: bool,
adapter_info: String,
}
fn drive_rebuild(device: &wgpu::Device, queue: &wgpu::Queue) -> Result<Measurements, String> {
let registry = GpuLossRegistry::new();
let packer = Box::new(Nv12Packer::new(device, PACKER_MAX_WIDTH, PACKER_MAX_HEIGHT));
let staging = Box::new(WgpuStagingBackend::new(
device,
WgpuStagingConfig::new(STAGING_BYTE_LEN),
));
let packer_built_before = GpuLossCallback::is_ready(packer.as_ref());
let staging_built_before = GpuLossCallback::is_ready(staging.as_ref());
assert!(packer_built_before, "packer must be ready before rebuild");
assert!(staging_built_before, "staging must be ready before rebuild");
let _packer_guard = registry.register(packer);
let _staging_guard = registry.register(staging);
let report = registry.handle_device_lost(device, queue);
assert!(
report.released_count as usize <= 2,
"no more than two owners released"
);
let mut packer_ready = false;
let mut staging_ready = false;
for outcome in report.outcomes.iter() {
match outcome {
fluxer_gpu_rebuild::RebuildOutcome::Rebuilt { label, .. } => {
if label.contains("nv12") {
packer_ready = true;
}
if label.contains("staging") {
staging_ready = true;
}
}
fluxer_gpu_rebuild::RebuildOutcome::Failed { .. } => {
return Err("rebuild outcome reported failure".to_string());
}
fluxer_gpu_rebuild::RebuildOutcome::Vacant { .. } => {}
}
}
Ok(Measurements {
released_count: report.released_count,
rebuilt_count: report.rebuilt_count,
failed_count: report.failed_count,
vacant_count: report.vacant_count,
packer_ready_after_rebuild: packer_ready,
staging_ready_after_rebuild: staging_ready,
adapter_info: format!("{:?}", device.limits().max_texture_dimension_2d),
})
}
@@ -0,0 +1,78 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
pub mod audio_capture_to_mix;
pub mod encoder_handoff_dryrun;
pub mod ffi_airlock_negative;
pub mod gpu_device_loss_recovery;
pub mod screen_capture_to_pool;
use serde::Serialize;
pub const SCHEMA_VERSION: u32 = 1;
#[derive(Debug, Serialize)]
pub struct ScenarioReport {
pub schema: u32,
pub scenario: &'static str,
pub platform: &'static str,
pub status: &'static str,
pub measurements: serde_json::Value,
pub assertions: Vec<String>,
}
impl ScenarioReport {
pub fn pass(
scenario: &'static str,
measurements: serde_json::Value,
assertions: Vec<String>,
) -> Self {
let report = Self {
schema: SCHEMA_VERSION,
scenario,
platform: current_platform(),
status: "pass",
measurements,
assertions,
};
assert_eq!(report.status, "pass", "pass constructor status invariant");
assert_eq!(report.schema, SCHEMA_VERSION, "schema version stable");
report
}
pub fn fail(
scenario: &'static str,
measurements: serde_json::Value,
assertions: Vec<String>,
) -> Self {
let report = Self {
schema: SCHEMA_VERSION,
scenario,
platform: current_platform(),
status: "fail",
measurements,
assertions,
};
assert_eq!(report.status, "fail", "fail constructor status invariant");
assert_eq!(report.schema, SCHEMA_VERSION, "schema version stable");
report
}
}
impl std::fmt::Display for ScenarioReport {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let serialised = serde_json::to_string(self).map_err(|_| std::fmt::Error)?;
f.write_str(&serialised)
}
}
pub const fn current_platform() -> &'static str {
if cfg!(target_os = "macos") {
"macos"
} else if cfg!(target_os = "linux") {
"linux"
} else if cfg!(target_os = "windows") {
"windows"
} else {
"unknown"
}
}
@@ -0,0 +1,100 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use fluxer_screen_frame_bus::frame_pool::{CpuFrameBuilder, FramePool};
use serde_json::json;
use super::ScenarioReport;
const FRAME_WIDTH_PX: u32 = 1920;
const FRAME_HEIGHT_PX: u32 = 1080;
const FRAME_BYTES_PER_PIXEL: u32 = 4;
const STEADY_STATE_FRAMES: u32 = 64;
pub fn run(_args: &[&str]) -> Result<ScenarioReport, ScenarioReport> {
let bytes_per_slot = (FRAME_WIDTH_PX as usize)
.saturating_mul(FRAME_HEIGHT_PX as usize)
.saturating_mul(FRAME_BYTES_PER_PIXEL as usize);
assert!(bytes_per_slot > 0, "bytes per slot positive");
assert!(
bytes_per_slot <= 1 << 28,
"bytes per slot within sanity cap"
);
let pool = CpuFrameBuilder::build_pool(bytes_per_slot).map_err(|err| {
ScenarioReport::fail(
"screen_capture_to_pool",
json!({"reason": format!("pool init failed: {err:?}")}),
Vec::new(),
)
})?;
let measurements = drive_steady_state(&pool);
let mut assertions = Vec::new();
if measurements.steady_state_skipped != 0 {
return Err(ScenarioReport::fail(
"screen_capture_to_pool",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec![format!(
"steady state should not skip; observed {} skips",
measurements.steady_state_skipped
)],
));
}
assertions.push("FramePool steady state shows zero skips".to_string());
if measurements.in_flight_after_release != 0 {
return Err(ScenarioReport::fail(
"screen_capture_to_pool",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec![format!(
"expected in-flight to drop to zero after release; observed {}",
measurements.in_flight_after_release
)],
));
}
assertions.push("PooledFrame Drop returns slot to free list".to_string());
if measurements.acquired_total_post_run < STEADY_STATE_FRAMES as u64 {
return Err(ScenarioReport::fail(
"screen_capture_to_pool",
serde_json::to_value(&measurements).unwrap_or(json!({})),
vec!["acquired_total below expected".to_string()],
));
}
assertions.push("acquired_total matches steady-state frame count".to_string());
Ok(ScenarioReport::pass(
"screen_capture_to_pool",
serde_json::to_value(&measurements).unwrap_or(json!({})),
assertions,
))
}
#[derive(Debug, serde::Serialize)]
struct Measurements {
pool_capacity: usize,
steady_state_frames: u32,
steady_state_skipped: u64,
acquired_total_post_run: u64,
in_flight_after_release: u64,
bytes_per_slot: usize,
}
fn drive_steady_state(pool: &FramePool) -> Measurements {
let capacity = pool.capacity();
assert!(capacity > 0, "pool capacity positive");
let baseline_skipped = pool.skipped_total();
for _ in 0..STEADY_STATE_FRAMES {
let pooled = pool
.try_acquire()
.expect("steady-state acquire must succeed since prior frame dropped");
let _ = pooled.slot_index();
}
let in_flight = pool.currently_in_flight();
let skipped = pool.skipped_total().saturating_sub(baseline_skipped);
Measurements {
pool_capacity: capacity,
steady_state_frames: STEADY_STATE_FRAMES,
steady_state_skipped: skipped,
acquired_total_post_run: pool.acquired_total(),
in_flight_after_release: in_flight,
bytes_per_slot: ((FRAME_WIDTH_PX as usize)
.saturating_mul(FRAME_HEIGHT_PX as usize)
.saturating_mul(FRAME_BYTES_PER_PIXEL as usize)),
}
}
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
INTEGRATION_TRANSFER_LIB_LOADED=1
integration_transfer_b64_encode() {
local src_path="$1"
if [ ! -f "$src_path" ]; then
printf 'integration_transfer: missing source %s\n' "$src_path" >&2
return 64
fi
if command -v base64 >/dev/null 2>&1; then
base64 < "$src_path" | tr -d '\n'
else
printf 'integration_transfer: base64 not available\n' >&2
return 64
fi
}
integration_transfer_ssh_write_text() {
local ssh_target="$1"
local password="$2"
local dest_path="$3"
local src_path="$4"
local encoded
encoded="$(integration_transfer_b64_encode "$src_path")" || return $?
local powershell_cmd
powershell_cmd="\$bytes = [System.Convert]::FromBase64String('${encoded}'); [System.IO.File]::WriteAllBytes('${dest_path}', \$bytes)"
if command -v sshpass >/dev/null 2>&1; then
sshpass -p "$password" ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "powershell -NoProfile -Command \"${powershell_cmd}\""
else
ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "powershell -NoProfile -Command \"${powershell_cmd}\""
fi
}
integration_transfer_ssh_run() {
local ssh_target="$1"
local password="$2"
local command_line="$3"
if command -v sshpass >/dev/null 2>&1; then
sshpass -p "$password" ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "$command_line"
else
ssh -o StrictHostKeyChecking=accept-new "$ssh_target" "$command_line"
fi
}
integration_transfer_ssh_powershell() {
local ssh_target="$1"
local password="$2"
local powershell_script="$3"
integration_transfer_ssh_run "$ssh_target" "$password" "powershell -NoProfile -Command \"${powershell_script}\""
}
integration_transfer_prlctl_exec() {
local vm_name="$1"
local command_line="$2"
if ! command -v prlctl >/dev/null 2>&1; then
printf 'integration_transfer: prlctl not available\n' >&2
return 64
fi
prlctl exec "$vm_name" "$command_line"
}
+2
View File
@@ -0,0 +1,2 @@
*.json
latest-summary.txt
+183
View File
@@ -0,0 +1,183 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
PLATFORM=""
ONLY_SCENARIO=""
DRY_RUN="0"
for arg in "$@"; do
case "$arg" in
--platform=*) PLATFORM="${arg#--platform=}" ;;
--only=*) ONLY_SCENARIO="${arg#--only=}" ;;
--dry-run) DRY_RUN="1" ;;
-h|--help)
cat <<'EOF'
usage: runner.sh --platform=<macos|linux-fedora|windows-tailnet|windows-vm> [--only=<scenario>] [--dry-run]
--platform=NAME Required. Selects the harness matrix entry.
--only=NAME Optional. Run a single scenario (script basename without .sh).
--dry-run Print which scenarios would run without invoking them.
EOF
exit 0
;;
*)
printf 'runner.sh: unknown argument %s\n' "$arg" >&2
exit 64
;;
esac
done
if [ -z "$PLATFORM" ]; then
printf 'runner.sh: --platform=<name> is required\n' >&2
exit 64
fi
RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RESULTS_DIR="${RUNNER_DIR}/results"
SCENARIOS_DIR="${RUNNER_DIR}/scenarios"
mkdir -p "$RESULTS_DIR"
case "$PLATFORM" in
macos)
SCENARIO_LIST=(
scenario_audio_capture_to_mix
scenario_screen_capture_to_pool
scenario_gpu_device_loss_recovery
scenario_event_log_replay_determinism
scenario_ffi_airlock_negative_space
scenario_encoder_handoff_dryrun
)
;;
linux-fedora)
SCENARIO_LIST=(
scenario_audio_capture_to_mix
scenario_screen_capture_to_pool
scenario_gpu_device_loss_recovery
scenario_event_log_replay_determinism
scenario_ffi_airlock_negative_space
scenario_encoder_handoff_dryrun
)
;;
windows-tailnet)
SCENARIO_LIST=(
scenario_audio_capture_to_mix
scenario_screen_capture_to_pool
scenario_gpu_device_loss_recovery
scenario_event_log_replay_determinism
scenario_ffi_airlock_negative_space
scenario_encoder_handoff_dryrun
)
;;
windows-vm)
SCENARIO_LIST=(
scenario_audio_capture_to_mix
scenario_screen_capture_to_pool
scenario_event_log_replay_determinism
scenario_ffi_airlock_negative_space
scenario_encoder_handoff_dryrun
)
;;
*)
printf 'runner.sh: unknown platform %s; expected macos|linux-fedora|windows-tailnet|windows-vm\n' "$PLATFORM" >&2
exit 64
;;
esac
if [ -n "$ONLY_SCENARIO" ]; then
FILTERED=()
for s in "${SCENARIO_LIST[@]}"; do
if [ "$s" = "$ONLY_SCENARIO" ]; then
FILTERED+=("$s")
fi
done
SCENARIO_LIST=("${FILTERED[@]}")
if [ "${#SCENARIO_LIST[@]}" -eq 0 ]; then
printf 'runner.sh: scenario %s not in platform matrix for %s\n' "$ONLY_SCENARIO" "$PLATFORM" >&2
exit 64
fi
fi
TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
RESULT_FILE="${RESULTS_DIR}/${PLATFORM}-${TIMESTAMP}.json"
SUMMARY_FILE="${RESULTS_DIR}/latest-summary.txt"
if [ "$DRY_RUN" = "1" ]; then
printf 'Would run on platform=%s:\n' "$PLATFORM"
for s in "${SCENARIO_LIST[@]}"; do
printf ' %s\n' "$s"
done
exit 0
fi
PASSED=0
FAILED=0
RESULT_ENTRIES=()
SUMMARY_LINES=()
emit_result_entry() {
local scenario="$1"
local status="$2"
local body="$3"
local entry
entry="{\"scenario\":\"${scenario}\",\"status\":\"${status}\",\"output\":${body}}"
RESULT_ENTRIES+=("$entry")
}
for scenario in "${SCENARIO_LIST[@]}"; do
script="${SCENARIOS_DIR}/${scenario}.sh"
if [ ! -x "$script" ]; then
chmod +x "$script" 2>/dev/null || true
fi
if [ ! -f "$script" ]; then
emit_result_entry "$scenario" "missing" "{\"reason\":\"scenario script not found at ${script}\"}"
SUMMARY_LINES+=("[MISS] ${scenario}")
FAILED=$((FAILED + 1))
continue
fi
set +e
raw_output="$("$script" 2>&1)"
exit_code=$?
set -e
last_json_line="$(printf '%s\n' "$raw_output" | awk '/^\{.*\}$/ {last=$0} END {print last}')"
if [ -z "$last_json_line" ]; then
emit_result_entry "$scenario" "fail" "{\"reason\":\"scenario produced no JSON line\",\"raw\":$(jq -Rs . <<<"$raw_output" 2>/dev/null || echo '\"<unparseable>\"')}"
SUMMARY_LINES+=("[FAIL] ${scenario} (no json)")
FAILED=$((FAILED + 1))
continue
fi
if [ "$exit_code" -eq 0 ]; then
emit_result_entry "$scenario" "pass" "$last_json_line"
SUMMARY_LINES+=("[PASS] ${scenario}")
PASSED=$((PASSED + 1))
else
emit_result_entry "$scenario" "fail" "$last_json_line"
SUMMARY_LINES+=("[FAIL] ${scenario} (exit ${exit_code})")
FAILED=$((FAILED + 1))
fi
done
ENTRIES_JOINED="$(IFS=,; echo "${RESULT_ENTRIES[*]}")"
printf '{"schema":1,"platform":"%s","timestamp":"%s","passed":%d,"failed":%d,"results":[%s]}\n' \
"$PLATFORM" "$TIMESTAMP" "$PASSED" "$FAILED" "$ENTRIES_JOINED" > "$RESULT_FILE"
{
printf 'platform: %s\n' "$PLATFORM"
printf 'timestamp: %s\n' "$TIMESTAMP"
printf 'passed: %d\n' "$PASSED"
printf 'failed: %d\n' "$FAILED"
printf 'scenarios:\n'
for line in "${SUMMARY_LINES[@]}"; do
printf ' %s\n' "$line"
done
printf 'result_file: %s\n' "$RESULT_FILE"
} > "$SUMMARY_FILE"
cat "$SUMMARY_FILE"
if [ "$FAILED" -gt 0 ]; then
exit 1
fi
exit 0
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
if [ -z "${INTEGRATION_HARNESS_ROOT:-}" ]; then
SCENARIO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
INTEGRATION_HARNESS_ROOT="$(cd "${SCENARIO_DIR}/.." && pwd)"
fi
export INTEGRATION_HARNESS_ROOT
INTEGRATION_REPO_ROOT="$(cd "${INTEGRATION_HARNESS_ROOT}/../.." && pwd)"
export INTEGRATION_REPO_ROOT
detect_platform() {
local uname_out
uname_out="$(uname -s 2>/dev/null || echo unknown)"
case "$uname_out" in
Darwin) echo "macos" ;;
Linux) echo "linux" ;;
MINGW*|MSYS*|CYGWIN*|Windows_NT) echo "windows" ;;
*)
if [ -n "${OS:-}" ] && [ "$OS" = "Windows_NT" ]; then
echo "windows"
else
echo "unknown"
fi
;;
esac
}
driver_target_dir() {
local platform="$1"
echo "${INTEGRATION_HARNESS_ROOT}/integration-driver/target-${platform}"
}
driver_binary_path() {
local platform="$1"
local target_dir
target_dir="$(driver_target_dir "$platform")"
if [ "$platform" = "windows" ]; then
echo "${target_dir}/debug/integration-driver.exe"
else
echo "${target_dir}/debug/integration-driver"
fi
}
driver_binary_runs() {
local binary="$1"
if [ ! -f "$binary" ]; then
return 1
fi
if "$binary" --help >/dev/null 2>&1; then
return 0
fi
if "$binary" 2>/dev/null | head -c 1 >/dev/null; then
return 0
fi
return 1
}
ensure_driver_built() {
local platform
platform="$(detect_platform)"
local binary
binary="$(driver_binary_path "$platform")"
local target_dir
target_dir="$(driver_target_dir "$platform")"
if ! driver_binary_runs "$binary"; then
( cd "${INTEGRATION_HARNESS_ROOT}/integration-driver" && CARGO_TARGET_DIR="$target_dir" cargo build >&2 ) || return $?
fi
if [ ! -f "$binary" ]; then
printf '{"schema":1,"scenario":"%s","platform":"%s","status":"fail","measurements":{"reason":"driver binary missing after build"},"assertions":[]}\n' \
"${SCENARIO_NAME:-unknown}" "$platform"
return 70
fi
echo "$binary"
}
run_rust_subcommand() {
local subcommand="$1"
shift || true
local binary
binary="$(ensure_driver_built)" || return $?
"$binary" "$subcommand" "$@"
}
run_ts_driver() {
local entry="$1"
shift || true
local resolved="${INTEGRATION_REPO_ROOT}/${entry}"
if [ ! -f "$resolved" ]; then
printf '{"schema":1,"scenario":"%s","platform":"%s","status":"fail","measurements":{"reason":"ts driver entry missing: %s"},"assertions":[]}\n' \
"${SCENARIO_NAME:-unknown}" "$(detect_platform)" "$resolved"
return 70
fi
local pnpm_command
pnpm_command="$(resolve_pnpm_command)" || {
printf '{"schema":1,"scenario":"%s","platform":"%s","status":"fail","measurements":{"reason":"pnpm not available"},"assertions":[]}\n' \
"${SCENARIO_NAME:-unknown}" "$(detect_platform)"
return 70
}
if should_use_dlx_tsx; then
( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command dlx tsx "$entry" "$@" )
elif ( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command exec tsx --version >/dev/null 2>&1 ); then
( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command exec tsx "$entry" "$@" )
else
( cd "$INTEGRATION_REPO_ROOT" && $pnpm_command dlx tsx "$entry" "$@" )
fi
}
resolve_pnpm_command() {
if command -v pnpm >/dev/null 2>&1; then
echo "pnpm"
return 0
fi
if command -v corepack >/dev/null 2>&1; then
echo "corepack pnpm"
return 0
fi
return 1
}
should_use_dlx_tsx() {
if [ "${FLUXER_INTEGRATION_TSX_MODE:-}" = "dlx" ]; then
return 0
fi
case "$INTEGRATION_REPO_ROOT" in
/media/psf/*|/mnt/psf/*)
return 0
;;
*)
return 1
;;
esac
}
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
SCENARIO_NAME="scenario_audio_capture_to_mix"
export SCENARIO_NAME
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
run_rust_subcommand "audio-capture-to-mix"
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
SCENARIO_NAME="scenario_encoder_handoff_dryrun"
export SCENARIO_NAME
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
run_rust_subcommand "encoder-handoff-dryrun"
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
SCENARIO_NAME="scenario_event_log_replay_determinism"
export SCENARIO_NAME
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
EXPECTED_FILE="${INTEGRATION_HARNESS_ROOT}/expected/event_log_replay_determinism.json"
EXPECTED_HASH=""
if [ -f "$EXPECTED_FILE" ]; then
EXPECTED_HASH="$(grep -E '"snapshot_hash"' "$EXPECTED_FILE" | head -1 | sed -E 's/.*"snapshot_hash"[^"]*"([^"]+)".*/\1/' || true)"
fi
if [ -n "$EXPECTED_HASH" ]; then
run_ts_driver "tools/integration/ts-driver/event_log_replay_determinism.ts" "$EXPECTED_HASH"
else
run_ts_driver "tools/integration/ts-driver/event_log_replay_determinism.ts"
fi
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
SCENARIO_NAME="scenario_ffi_airlock_negative_space"
export SCENARIO_NAME
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
run_rust_subcommand "ffi-airlock-negative" "invalid-sample-rate"
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
SCENARIO_NAME="scenario_gpu_device_loss_recovery"
export SCENARIO_NAME
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
run_rust_subcommand "gpu-device-loss-recovery"
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
set -u
set -o pipefail
SCENARIO_NAME="scenario_screen_capture_to_pool"
export SCENARIO_NAME
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
run_rust_subcommand "screen-capture-to-pool"
@@ -0,0 +1,118 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {VoiceEngineV2Simulator} from '../../../packages/voice_engine_v2/src/simulation/Simulator';
import {defineNetworkPartitionDuringScreenShareScenario} from '../../../packages/voice_engine_v2/src/simulation/scenarios/networkPartitionDuringScreenShare';
const REPORT_SCHEMA = 1;
const SCENARIO_NAME = 'event_log_replay_determinism';
const SEED = 4242;
interface Report {
schema: number;
scenario: string;
platform: string;
status: 'pass' | 'fail';
measurements: Record<string, unknown>;
assertions: Array<string>;
}
function platform(): string {
if (process.platform === 'darwin') return 'macos';
if (process.platform === 'linux') return 'linux';
if (process.platform === 'win32') return 'windows';
return 'unknown';
}
async function runOnce(): Promise<{snapshotHash: string; eventCount: number; finalTick: number}> {
const scenario = defineNetworkPartitionDuringScreenShareScenario(SEED);
assert.ok(scenario.workload, 'scenario must yield a workload');
const simulator = new VoiceEngineV2Simulator({
seed: SEED,
workload: scenario.workload,
faults: scenario.faultPlan,
mode: scenario.mode,
});
const result = await simulator.run();
assert.ok(typeof result.snapshotHash === 'string', 'snapshot hash must be a string');
assert.equal(result.snapshotHash.length, 8, 'snapshot hash is eight hex characters');
return {
snapshotHash: result.snapshotHash,
eventCount: result.eventLog.length,
finalTick: result.finalTick,
};
}
async function main(): Promise<void> {
const expectedHashArg = process.argv[2] ?? null;
const first = await runOnce();
const second = await runOnce();
const assertions: Array<string> = [];
if (first.snapshotHash !== second.snapshotHash) {
emit({
schema: REPORT_SCHEMA,
scenario: SCENARIO_NAME,
platform: platform(),
status: 'fail',
measurements: {first, second},
assertions: ['two runs of the same scenario must agree on snapshot hash'],
});
process.exit(1);
}
assertions.push('two runs of the same scenario agree on snapshot hash');
if (first.eventCount !== second.eventCount) {
emit({
schema: REPORT_SCHEMA,
scenario: SCENARIO_NAME,
platform: platform(),
status: 'fail',
measurements: {first, second},
assertions: ['event log length must agree across runs'],
});
process.exit(1);
}
assertions.push('event log length agrees across runs');
if (expectedHashArg !== null && expectedHashArg !== first.snapshotHash) {
emit({
schema: REPORT_SCHEMA,
scenario: SCENARIO_NAME,
platform: platform(),
status: 'fail',
measurements: {expected: expectedHashArg, observed: first.snapshotHash, runs: [first, second]},
assertions: [`snapshot hash mismatch: expected ${expectedHashArg}, observed ${first.snapshotHash}`],
});
process.exit(1);
}
if (expectedHashArg !== null) {
assertions.push('snapshot hash matches expected fixture');
}
emit({
schema: REPORT_SCHEMA,
scenario: SCENARIO_NAME,
platform: platform(),
status: 'pass',
measurements: {
snapshot_hash: first.snapshotHash,
event_count: first.eventCount,
final_tick: first.finalTick,
seed: SEED,
},
assertions,
});
}
function emit(report: Report): void {
process.stdout.write(`${JSON.stringify(report)}\n`);
}
main().catch((error) => {
emit({
schema: REPORT_SCHEMA,
scenario: SCENARIO_NAME,
platform: platform(),
status: 'fail',
measurements: {error: String(error)},
assertions: ['driver threw before finishing scenario'],
});
process.exit(1);
});
@@ -0,0 +1,14 @@
[package]
name = "fluxer-marketing-update-gettext-catalogs"
version = "0.1.0"
edition.workspace = true
license.workspace = true
[dependencies]
anyhow = "1.0.102"
chrono = "0.4.45"
serde_json = "1.0.150"
syn = { version = "2.0.117", features = ["full", "visit"] }
[dev-dependencies]
tempfile = "3.27.0"
@@ -0,0 +1,660 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::{Context, Result, anyhow, bail};
use chrono::{Local, NaiveDate};
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use syn::{
Expr, ExprLit, ExprMacro, Ident, Lit, Macro, Token, braced,
parse::{Parse, ParseStream},
visit::{self, Visit},
};
#[derive(Clone, Debug, PartialEq, Eq)]
struct Descriptor {
const_name: String,
key: String,
message: String,
comment: String,
}
pub fn find_marketing_root(current_dir: &Path) -> Result<PathBuf> {
for ancestor in current_dir.ancestors() {
if is_marketing_root(ancestor) {
return Ok(ancestor.to_path_buf());
}
let child = ancestor.join("fluxer_marketing");
if is_marketing_root(&child) {
return Ok(child);
}
}
bail!(
"could not find fluxer_marketing from {}; run from the repository root or pass fluxer_marketing explicitly",
current_dir.display()
);
}
pub fn update_catalogs(root: &Path) -> Result<()> {
update_catalogs_with_date(root, Local::now().date_naive())
}
pub fn update_catalogs_with_date(root: &Path, today: NaiveDate) -> Result<()> {
let descriptors = parse_descriptors(root)?;
let locales_dir = root.join("locales");
if !locales_dir.is_dir() {
return Ok(());
}
let mut locales = fs::read_dir(&locales_dir)
.with_context(|| format!("failed to read {}", locales_dir.display()))?
.map(|entry| entry.map(|entry| entry.path()))
.collect::<std::io::Result<Vec<_>>>()
.with_context(|| format!("failed to read {}", locales_dir.display()))?
.into_iter()
.filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("po"))
.filter_map(|path| {
path.file_stem()
.and_then(|stem| stem.to_str())
.map(str::to_owned)
})
.collect::<Vec<_>>();
locales.sort();
for locale in locales {
write_catalog(&locales_dir, &locale, &descriptors, today)?;
}
Ok(())
}
fn is_marketing_root(path: &Path) -> bool {
path.join("src/i18n/descriptors.rs").is_file() && path.join("locales").is_dir()
}
fn parse_descriptors(root: &Path) -> Result<Vec<Descriptor>> {
let descriptors_path = root.join("src/i18n/descriptors.rs");
let descriptors_dir = root.join("src/i18n/descriptors");
let mut descriptors = Vec::new();
for path in descriptor_sources(&descriptors_path, &descriptors_dir)? {
let source = fs::read_to_string(&path)
.with_context(|| format!("failed to read {}", path.display()))?;
let file = syn::parse_file(&source)
.with_context(|| format!("failed to parse {}", path.display()))?;
let mut visitor = MarketingMessageVisitor::new(&path);
visitor.visit_file(&file);
for descriptor in visitor.descriptors {
descriptors.push(descriptor?);
}
}
if descriptors.is_empty() {
bail!(
"no descriptors found under {} / {}",
descriptors_path.display(),
descriptors_dir.display()
);
}
Ok(descriptors)
}
fn descriptor_sources(descriptors_path: &Path, descriptors_dir: &Path) -> Result<Vec<PathBuf>> {
let mut sources = vec![descriptors_path.to_path_buf()];
if descriptors_dir.is_dir() {
let mut children = fs::read_dir(descriptors_dir)
.with_context(|| format!("failed to read {}", descriptors_dir.display()))?
.map(|entry| entry.map(|entry| entry.path()))
.collect::<std::io::Result<Vec<_>>>()
.with_context(|| format!("failed to read {}", descriptors_dir.display()))?
.into_iter()
.filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("rs"))
.collect::<Vec<_>>();
children.sort();
sources.extend(children);
}
Ok(sources)
}
struct MarketingMessageVisitor<'a> {
descriptor_file: &'a Path,
descriptors: Vec<Result<Descriptor>>,
}
impl<'a> MarketingMessageVisitor<'a> {
fn new(descriptor_file: &'a Path) -> Self {
Self {
descriptor_file,
descriptors: Vec::new(),
}
}
}
impl<'ast> Visit<'ast> for MarketingMessageVisitor<'_> {
fn visit_macro(&mut self, node: &'ast Macro) {
if is_marketing_message_macro(node) {
let descriptor = node
.parse_body::<MarketingMessageInput>()
.with_context(|| {
format!(
"failed to parse marketing_message! descriptor in {}",
self.descriptor_file.display()
)
})
.and_then(|input| input.into_descriptor(self.descriptor_file));
self.descriptors.push(descriptor);
}
visit::visit_macro(self, node);
}
}
struct MarketingMessageInput {
const_name: Ident,
key: Expr,
message: Expr,
comment: Expr,
}
impl MarketingMessageInput {
fn into_descriptor(self, descriptor_file: &Path) -> Result<Descriptor> {
let const_name = self.const_name.to_string();
let key = expect_string_literal(&self.key, &const_name, "key", descriptor_file)?;
let message = parse_descriptor_message(descriptor_file, &const_name, &self.message)?;
let comment =
expect_string_literal(&self.comment, &const_name, "comment", descriptor_file)?;
Ok(Descriptor {
const_name,
key,
message,
comment,
})
}
}
impl Parse for MarketingMessageInput {
fn parse(input: ParseStream) -> syn::Result<Self> {
input.parse::<Token![pub]>()?;
input.parse::<Token![const]>()?;
let const_name = input.parse()?;
input.parse::<Token![=]>()?;
let body;
braced!(body in input);
let key = parse_expected_field(&body, "key")?;
body.parse::<Token![,]>()?;
let message = parse_expected_field(&body, "message")?;
body.parse::<Token![,]>()?;
let comment = parse_expected_field(&body, "comment")?;
if body.peek(Token![,]) {
body.parse::<Token![,]>()?;
}
if !body.is_empty() {
return Err(body.error("unexpected descriptor field"));
}
input.parse::<Token![;]>()?;
if !input.is_empty() {
return Err(input.error("unexpected tokens after descriptor"));
}
Ok(Self {
const_name,
key,
message,
comment,
})
}
}
fn parse_expected_field(input: ParseStream, expected_name: &str) -> syn::Result<Expr> {
let name: Ident = input.parse()?;
if name != expected_name {
return Err(syn::Error::new(
name.span(),
format!("expected `{expected_name}` field"),
));
}
input.parse::<Token![:]>()?;
input.parse()
}
fn is_marketing_message_macro(node: &Macro) -> bool {
if node.path.leading_colon.is_some() {
return false;
}
let segments = node
.path
.segments
.iter()
.map(|segment| segment.ident.to_string())
.collect::<Vec<_>>();
matches!(segments.as_slice(), [name] if name == "marketing_message")
|| matches!(segments.as_slice(), [root, name] if root == "crate" && name == "marketing_message")
}
fn expect_string_literal(
expr: &Expr,
const_name: &str,
field_name: &str,
descriptor_file: &Path,
) -> Result<String> {
if let Expr::Lit(ExprLit {
lit: Lit::Str(value),
..
}) = expr
{
return Ok(value.value());
}
bail!(
"descriptor {} field {} must be a string literal in {}",
const_name,
field_name,
descriptor_file.display()
);
}
fn parse_descriptor_message(
descriptor_file: &Path,
const_name: &str,
message: &Expr,
) -> Result<String> {
if let Expr::Lit(ExprLit {
lit: Lit::Str(value),
..
}) = message
{
return Ok(value.value());
}
let Expr::Macro(ExprMacro { mac, .. }) = message else {
bail!(
"descriptor {} message must be a string literal or include_str!(...) in {}",
const_name,
descriptor_file.display()
);
};
if !is_include_str_macro(mac) {
bail!(
"descriptor {} message must be a string literal or include_str!(...) in {}",
const_name,
descriptor_file.display()
);
}
let relative_path = mac.parse_body::<syn::LitStr>().with_context(|| {
format!(
"invalid include_str! descriptor message in {} for {}",
descriptor_file.display(),
const_name
)
})?;
let source_path = descriptor_file
.parent()
.ok_or_else(|| anyhow!("descriptor file should have a parent directory"))?
.join(relative_path.value());
fs::read_to_string(&source_path)
.with_context(|| format!("failed to read descriptor source {}", source_path.display()))
}
fn is_include_str_macro(node: &Macro) -> bool {
node.path.leading_colon.is_none()
&& node.path.segments.len() == 1
&& node.path.segments[0].ident == "include_str"
}
fn parse_existing_translations(path: &Path) -> Result<BTreeMap<String, String>> {
if !path.exists() {
return Ok(BTreeMap::new());
}
let text =
fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?;
let mut translations = BTreeMap::new();
for entry in split_po_entries(&text) {
let context = parse_po_field(entry, "msgctxt")?;
let msgstr = parse_po_field(entry, "msgstr")?;
if let (Some(context), Some(msgstr)) = (context, msgstr) {
translations.insert(context, msgstr);
}
}
Ok(translations)
}
fn split_po_entries(text: &str) -> Vec<&str> {
let lines = text.split_inclusive('\n').collect::<Vec<_>>();
let mut entries = Vec::new();
let mut start = 0;
let mut offset = 0;
for line in lines {
let line_start = offset;
offset += line.len();
if line.trim().is_empty() {
entries.push(&text[start..line_start]);
start = offset;
}
}
if start <= text.len() {
entries.push(&text[start..]);
}
entries
}
fn parse_po_field(entry: &str, field: &str) -> Result<Option<String>> {
let prefix = format!("{field} ");
let lines = entry.lines().collect::<Vec<_>>();
for (index, line) in lines.iter().enumerate() {
let Some(value) = line.strip_prefix(&prefix) else {
continue;
};
let mut values = vec![value];
for continuation in lines.iter().skip(index + 1) {
if !continuation.starts_with('"') {
break;
}
values.push(continuation);
}
let mut parsed = String::new();
for value in values {
parsed.push_str(&unescape_po_string(value)?);
}
return Ok(Some(parsed));
}
Ok(None)
}
fn unescape_po_string(quoted: &str) -> Result<String> {
if let Ok(value) = serde_json::from_str::<String>(quoted) {
return Ok(value);
}
let stripped = quoted.trim();
if !stripped.starts_with('"') || !stripped.ends_with('"') {
bail!("not a PO quoted string: {quoted:?}");
}
let inner = &stripped[1..stripped.len() - 1];
let mut result = String::new();
let mut chars = inner.chars();
while let Some(ch) = chars.next() {
if ch == '\\' {
if let Some(next) = chars.next() {
match next {
'n' => result.push('\n'),
't' => result.push('\t'),
'\\' => result.push('\\'),
'"' => result.push('"'),
other => {
result.push(ch);
result.push(other);
}
}
} else {
result.push(ch);
}
} else {
result.push(ch);
}
}
Ok(result)
}
fn write_catalog(
locales_dir: &Path,
locale: &str,
descriptors: &[Descriptor],
today: NaiveDate,
) -> Result<()> {
let path = locales_dir.join(format!("{locale}.po"));
let existing = parse_existing_translations(&path)?;
let today = today.to_string();
let mut lines = vec![
"# SPDX-License-Identifier: AGPL-3.0-or-later".to_owned(),
"#".to_owned(),
format!("# Fluxer marketing gettext catalog for {locale}."),
"msgid \"\"".to_owned(),
"msgstr \"\"".to_owned(),
"\"Project-Id-Version: fluxer-marketing\\n\"".to_owned(),
format!("\"POT-Creation-Date: {today} 00:00+0000\\n\""),
format!("\"PO-Revision-Date: {today} 00:00+0000\\n\""),
format!("\"Language: {locale}\\n\""),
"\"MIME-Version: 1.0\\n\"".to_owned(),
"\"Content-Type: text/plain; charset=UTF-8\\n\"".to_owned(),
"\"Content-Transfer-Encoding: 8bit\\n\"".to_owned(),
"\"Plural-Forms: nplurals=2; plural=(n != 1);\\n\"".to_owned(),
String::new(),
];
for descriptor in descriptors {
let translated = if locale == "en-US" {
descriptor.message.as_str()
} else {
existing
.get(&descriptor.key)
.map(String::as_str)
.unwrap_or(&descriptor.message)
};
lines.push(format!("#. {}", descriptor.comment));
lines.push(format!("#: fluxer_marketing/generated:{}", descriptor.key));
write_field(&mut lines, "msgctxt", &descriptor.key);
write_field(&mut lines, "msgid", &descriptor.message);
write_field(&mut lines, "msgstr", translated);
lines.push(String::new());
}
fs::write(&path, lines.join("\n"))
.with_context(|| format!("failed to write {}", path.display()))
}
fn po_escape(value: &str) -> String {
value
.replace('\\', "\\\\")
.replace('"', "\\\"")
.replace('\n', "\\n")
}
fn write_field(lines: &mut Vec<String>, name: &str, value: &str) {
if !value.contains('\n') && value.chars().count() <= 90 {
lines.push(format!(r#"{name} "{}""#, po_escape(value)));
return;
}
lines.push(format!(r#"{name} """#));
for part in split_lines_keepends(value) {
lines.push(format!(r#""{}""#, po_escape(part)));
}
}
fn split_lines_keepends(value: &str) -> Vec<&str> {
let mut parts = Vec::new();
let mut start = 0;
for (index, ch) in value.char_indices() {
if ch == '\n' {
let end = index + ch.len_utf8();
parts.push(&value[start..end]);
start = end;
}
}
if start < value.len() {
parts.push(&value[start..]);
}
if parts.is_empty() {
parts.push("");
}
parts
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::NaiveDate;
use std::fs;
use tempfile::tempdir;
#[test]
fn updates_catalogs_from_literals_and_include_str_sources() {
let dir = tempdir().unwrap();
let root = dir.path().join("fluxer_marketing");
fs::create_dir_all(root.join("src/i18n/descriptors")).unwrap();
fs::create_dir_all(root.join("locales")).unwrap();
fs::write(
root.join("src/i18n/descriptors.rs"),
r#"
marketing_message!(
pub const SHORT_DESCRIPTOR = {
key: "app.short",
message: "Hello \"Fluxer\"",
comment: "Shown on the fixture page with escaped quotes for translators.",
};
);
"#,
)
.unwrap();
fs::write(
root.join("src/i18n/descriptors/extra.rs"),
r#"
crate::marketing_message!(
pub const BODY_DESCRIPTOR = {
key: "content.body",
message: include_str!("body.txt"),
comment: "Body copy loaded from a markdown fixture and shown on a content page.",
};
);
"#,
)
.unwrap();
fs::write(
root.join("src/i18n/descriptors/body.txt"),
"Body first line\nBody second \"quote\" \\ path\n",
)
.unwrap();
fs::write(
root.join("locales/en-US.po"),
r#"
msgctxt "app.short"
msgid "Old"
msgstr "Outdated"
"#,
)
.unwrap();
fs::write(
root.join("locales/sv-SE.po"),
r#"
msgctxt "app.short"
msgid "Old"
msgstr "Preserved short"
msgctxt "content.body"
msgid "Old body"
msgstr ""
"Preserved line one\n"
"Preserved \"quote\" \\ path\n"
msgctxt "obsolete"
msgid "Obsolete"
msgstr "Should disappear"
"#,
)
.unwrap();
update_catalogs_with_date(
&root,
NaiveDate::from_ymd_opt(2026, 6, 4).expect("valid date"),
)
.unwrap();
let en_us_path = root.join("locales/en-US.po");
let sv_se_path = root.join("locales/sv-SE.po");
let en_us_text = fs::read_to_string(&en_us_path).unwrap();
let sv_se_text = fs::read_to_string(&sv_se_path).unwrap();
assert!(en_us_text.contains("\"POT-Creation-Date: 2026-06-04 00:00+0000\\n\""));
assert!(en_us_text.contains("msgstr \"Hello \\\"Fluxer\\\"\""));
assert!(en_us_text.contains(
r#"msgid ""
"Body first line\n"
"Body second \"quote\" \\ path\n""#
));
assert!(sv_se_text.contains("msgstr \"Preserved short\""));
assert!(!sv_se_text.contains("obsolete"));
let en_us = parse_existing_translations(&en_us_path).unwrap();
let sv_se = parse_existing_translations(&sv_se_path).unwrap();
assert_eq!(en_us["app.short"], "Hello \"Fluxer\"");
assert_eq!(
en_us["content.body"],
"Body first line\nBody second \"quote\" \\ path\n"
);
assert_eq!(sv_se["app.short"], "Preserved short");
assert_eq!(
sv_se["content.body"],
"Preserved line one\nPreserved \"quote\" \\ path\n"
);
}
#[test]
fn writes_long_single_line_fields_as_multiline_po_strings() {
let value = "x".repeat(91);
let mut lines = Vec::new();
write_field(&mut lines, "msgid", &value);
assert_eq!(lines, vec!["msgid \"\"".to_owned(), format!("\"{value}\"")]);
}
#[test]
fn unescapes_unknown_po_escapes_losslessly() {
assert_eq!(unescape_po_string(r#""a\qb""#).unwrap(), r#"a\qb"#);
assert_eq!(
unescape_po_string(r#""line\nquote\"slash\\""#).unwrap(),
"line\nquote\"slash\\"
);
}
#[test]
fn reports_missing_include_str_source() {
let dir = tempdir().unwrap();
let root = dir.path().join("fluxer_marketing");
fs::create_dir_all(root.join("src/i18n/descriptors")).unwrap();
fs::create_dir_all(root.join("locales")).unwrap();
fs::write(root.join("locales/en-US.po"), "").unwrap();
fs::write(root.join("src/i18n/descriptors.rs"), "").unwrap();
fs::write(
root.join("src/i18n/descriptors/missing.rs"),
r#"
crate::marketing_message!(
pub const MISSING_DESCRIPTOR = {
key: "missing.body",
message: include_str!("missing.txt"),
comment: "Body copy loaded from a missing fixture file for failure handling.",
};
);
"#,
)
.unwrap();
let err = update_catalogs_with_date(
&root,
NaiveDate::from_ymd_opt(2026, 6, 4).expect("valid date"),
)
.unwrap_err()
.to_string();
assert!(err.contains("failed to read descriptor source"));
assert!(err.contains("missing.txt"));
}
#[test]
fn finds_marketing_root_from_repo_or_package_directory() {
let dir = tempdir().unwrap();
let repo = dir.path();
let root = repo.join("fluxer_marketing");
fs::create_dir_all(root.join("src/i18n")).unwrap();
fs::create_dir_all(root.join("locales")).unwrap();
fs::write(root.join("src/i18n/descriptors.rs"), "").unwrap();
let nested_tool_dir = repo.join("tools/marketing/update-gettext-catalogs");
fs::create_dir_all(&nested_tool_dir).unwrap();
assert_eq!(find_marketing_root(repo).unwrap(), root);
assert_eq!(
find_marketing_root(&repo.join("fluxer_marketing")).unwrap(),
root
);
assert_eq!(find_marketing_root(&nested_tool_dir).unwrap(), root);
}
}
@@ -0,0 +1,34 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use anyhow::Result;
use fluxer_marketing_update_gettext_catalogs::{find_marketing_root, update_catalogs};
use std::env;
use std::path::PathBuf;
use std::process;
fn main() {
if let Err(err) = run() {
eprintln!("error: {err:#}");
process::exit(1);
}
}
fn run() -> Result<()> {
let mut args = env::args_os();
let program = args
.next()
.and_then(|value| PathBuf::from(value).file_name().map(|name| name.to_owned()))
.and_then(|name| name.into_string().ok())
.unwrap_or_else(|| "fluxer-marketing-update-gettext-catalogs".to_owned());
let root = match (args.next(), args.next()) {
(None, None) => find_marketing_root(&env::current_dir()?)?,
(Some(root), None) => PathBuf::from(root),
_ => {
eprintln!("usage: {program} [marketing_root]");
process::exit(1);
}
};
update_catalogs(&root)
}