Add native self-hosted instance connection to fluxer_desktop
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
This commit is contained in:
Generated
+3436
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "fluxer-ci"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0-or-later"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.102"
|
||||
aws-config = "1.8.18"
|
||||
aws-sdk-s3 = "1.135.0"
|
||||
base64 = "0.22.1"
|
||||
bytes = "1.11.1"
|
||||
chrono = "0.4.45"
|
||||
clap = { version = "4.6.1", features = ["derive"] }
|
||||
flate2 = "1.1.9"
|
||||
hex = "0.4.3"
|
||||
md-5 = "0.11.0"
|
||||
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
sha2 = "0.11.0"
|
||||
tar = "0.4.46"
|
||||
tempfile = "3.27.0"
|
||||
tokio = { version = "1.52.3", features = ["fs", "io-util", "macros", "process", "rt-multi-thread", "signal", "sync", "time"] }
|
||||
walkdir = "2.5.0"
|
||||
zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
|
||||
@@ -0,0 +1,539 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::app_wasm::resolve_app_dir;
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use clap::Args;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::ffi::OsStr;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tokio::process::{Child, Command};
|
||||
use tokio::sync::watch;
|
||||
use tokio::time::timeout;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
const DEFAULT_SKIP_DIRS: &[&str] = &[".git", "node_modules", "dist", "target", "pkg", "pkgs"];
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct AppDevServerArgs {
|
||||
#[arg(long)]
|
||||
app_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct StepMetadata {
|
||||
last_run: f64,
|
||||
inputs: BTreeMap<String, f64>,
|
||||
}
|
||||
|
||||
type Metadata = BTreeMap<String, StepMetadata>;
|
||||
|
||||
pub async fn run(args: AppDevServerArgs) -> Result<()> {
|
||||
let project_root = args.app_dir.unwrap_or(resolve_app_dir()?);
|
||||
let mut server = AppDevServer::new(project_root);
|
||||
server.run().await
|
||||
}
|
||||
|
||||
struct AppDevServer {
|
||||
project_root: PathBuf,
|
||||
metadata_file: PathBuf,
|
||||
metadata: Metadata,
|
||||
}
|
||||
|
||||
impl AppDevServer {
|
||||
fn new(project_root: PathBuf) -> Self {
|
||||
Self {
|
||||
metadata_file: project_root.join(".devserver-cache.json"),
|
||||
project_root,
|
||||
metadata: Metadata::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(&mut self) -> Result<()> {
|
||||
self.load_metadata();
|
||||
let (shutdown_tx, mut shutdown_rx) = watch::channel(false);
|
||||
tokio::spawn(listen_for_shutdown(shutdown_tx));
|
||||
|
||||
self.run_cached_step(
|
||||
"wasm",
|
||||
gather_wasm_inputs,
|
||||
"pnpm wasm:codegen",
|
||||
|server, shutdown| Box::pin(server.run_command("pnpm", &["wasm:codegen"], shutdown)),
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"colors",
|
||||
gather_color_inputs,
|
||||
"pnpm generate:colors",
|
||||
|server, shutdown| Box::pin(server.run_command("pnpm", &["generate:colors"], shutdown)),
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"messageLayout",
|
||||
gather_message_layout_inputs,
|
||||
"pnpm generate:message-layout",
|
||||
|server, shutdown| {
|
||||
Box::pin(server.run_command("pnpm", &["generate:message-layout"], shutdown))
|
||||
},
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"masks",
|
||||
gather_mask_inputs,
|
||||
"pnpm generate:masks",
|
||||
|server, shutdown| Box::pin(server.run_command("pnpm", &["generate:masks"], shutdown)),
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
self.run_cached_step(
|
||||
"cssTypes",
|
||||
gather_css_module_inputs,
|
||||
"pnpm generate:css-types",
|
||||
|server, shutdown| {
|
||||
Box::pin(server.run_command("pnpm", &["generate:css-types"], shutdown))
|
||||
},
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if env_truthy("FLUXER_APP_SKIP_I18N_COMPILE") {
|
||||
eprintln!("Skipping pnpm lingui:compile because FLUXER_APP_SKIP_I18N_COMPILE is set.");
|
||||
} else {
|
||||
self.run_cached_step(
|
||||
"lingui",
|
||||
gather_lingui_inputs,
|
||||
"pnpm lingui:compile",
|
||||
|server, shutdown| {
|
||||
Box::pin(server.run_command("pnpm", &["lingui:compile"], shutdown))
|
||||
},
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if *shutdown_rx.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.clean_dist()?;
|
||||
let mut css_type_watcher = self.start_css_type_watcher()?;
|
||||
let rspack_result = self.run_rspack(&mut shutdown_rx).await;
|
||||
terminate_child(&mut css_type_watcher).await;
|
||||
rspack_result
|
||||
}
|
||||
|
||||
fn load_metadata(&mut self) {
|
||||
match fs::read_to_string(&self.metadata_file) {
|
||||
Ok(raw) => match serde_json::from_str::<Metadata>(&raw) {
|
||||
Ok(metadata) => {
|
||||
self.metadata = metadata;
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!(
|
||||
"Failed to parse dev server metadata cache, falling back to full rebuild: {error}"
|
||||
);
|
||||
self.metadata = Metadata::default();
|
||||
}
|
||||
},
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
self.metadata = Metadata::default();
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!(
|
||||
"Failed to read dev server metadata cache, falling back to full rebuild: {error}"
|
||||
);
|
||||
self.metadata = Metadata::default();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn save_metadata(&self) -> Result<()> {
|
||||
if let Some(parent) = self.metadata_file.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
fs::write(
|
||||
&self.metadata_file,
|
||||
serde_json::to_string_pretty(&self.metadata)?,
|
||||
)
|
||||
.with_context(|| format!("Failed to write {}", self.metadata_file.display()))
|
||||
}
|
||||
|
||||
async fn run_cached_step<G, E>(
|
||||
&mut self,
|
||||
step_name: &'static str,
|
||||
gather_inputs: G,
|
||||
label: &'static str,
|
||||
execute: E,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<()>
|
||||
where
|
||||
G: Fn(&Path) -> Result<BTreeMap<String, f64>>,
|
||||
E: for<'a> FnOnce(
|
||||
&'a AppDevServer,
|
||||
&'a mut watch::Receiver<bool>,
|
||||
)
|
||||
-> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>>,
|
||||
{
|
||||
let inputs = gather_inputs(&self.project_root)?;
|
||||
if !self.should_run_step(step_name, &inputs) {
|
||||
println!("Skipping {label} (no changes detected)");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
execute(self, shutdown).await?;
|
||||
self.metadata.insert(
|
||||
step_name.to_string(),
|
||||
StepMetadata {
|
||||
last_run: timestamp_ms(SystemTime::now())?,
|
||||
inputs,
|
||||
},
|
||||
);
|
||||
self.save_metadata()
|
||||
}
|
||||
|
||||
fn should_run_step(&self, step_name: &str, inputs: &BTreeMap<String, f64>) -> bool {
|
||||
let Some(entry) = self.metadata.get(step_name) else {
|
||||
return true;
|
||||
};
|
||||
&entry.inputs != inputs
|
||||
}
|
||||
|
||||
async fn run_command(
|
||||
&self,
|
||||
command: &str,
|
||||
args: &[&str],
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<()> {
|
||||
if *shutdown.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut child = spawn_child(command, args, &self.project_root)?;
|
||||
let status = wait_for_child(command, args, &mut child, shutdown).await?;
|
||||
if *shutdown.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
ensure!(
|
||||
status.success(),
|
||||
"{} exited with status {}",
|
||||
display_command(command, args),
|
||||
status.code().unwrap_or(1)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn clean_dist(&self) -> Result<()> {
|
||||
let dist_path = self.project_root.join("dist");
|
||||
match fs::remove_dir_all(&dist_path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => {
|
||||
Err(error).with_context(|| format!("Failed to remove {}", dist_path.display()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn start_css_type_watcher(&self) -> Result<Child> {
|
||||
let tcm = self.project_root.join("node_modules/.bin/tcm");
|
||||
println!(
|
||||
"+ {} src --pattern '**/*.module.css' --watch --silent",
|
||||
tcm.display()
|
||||
);
|
||||
Command::new(tcm)
|
||||
.args(["src", "--pattern", "**/*.module.css", "--watch", "--silent"])
|
||||
.current_dir(&self.project_root)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.context("Failed to start CSS type watcher")
|
||||
}
|
||||
|
||||
async fn run_rspack(&self, shutdown: &mut watch::Receiver<bool>) -> Result<()> {
|
||||
let rspack = self.project_root.join("node_modules/.bin/rspack");
|
||||
let rspack_string = rspack.to_string_lossy().to_string();
|
||||
let mut child = spawn_child(
|
||||
&rspack_string,
|
||||
&["serve", "--mode", "development"],
|
||||
&self.project_root,
|
||||
)?;
|
||||
let status = wait_for_child(
|
||||
&rspack_string,
|
||||
&["serve", "--mode", "development"],
|
||||
&mut child,
|
||||
shutdown,
|
||||
)
|
||||
.await?;
|
||||
if *shutdown.borrow() {
|
||||
return Ok(());
|
||||
}
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
bail!(
|
||||
"rspack serve exited with status {}",
|
||||
status.code().unwrap_or(1)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_file_stats(project_root: &Path, paths: &[PathBuf]) -> Result<BTreeMap<String, f64>> {
|
||||
let mut result = BTreeMap::new();
|
||||
for rel_path in paths {
|
||||
let absolute_path = project_root.join(rel_path);
|
||||
let metadata = fs::metadata(&absolute_path)
|
||||
.with_context(|| format!("Failed to stat {}", absolute_path.display()))?;
|
||||
ensure!(
|
||||
metadata.is_file(),
|
||||
"Expected {} to be a file when collecting dev server cache inputs.",
|
||||
rel_path.display()
|
||||
);
|
||||
result.insert(rel_path_key(rel_path), timestamp_ms(metadata.modified()?)?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn collect_directory_stats<P>(
|
||||
project_root: &Path,
|
||||
root_rel: &Path,
|
||||
predicate: P,
|
||||
) -> Result<BTreeMap<String, f64>>
|
||||
where
|
||||
P: Fn(&str) -> bool,
|
||||
{
|
||||
let skip_dirs: BTreeSet<&str> = DEFAULT_SKIP_DIRS.iter().copied().collect();
|
||||
let root = project_root.join(root_rel);
|
||||
let mut result = BTreeMap::new();
|
||||
if !root.exists() {
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
for entry in WalkDir::new(&root)
|
||||
.into_iter()
|
||||
.filter_entry(|entry| should_walk_entry(entry.path(), &skip_dirs))
|
||||
{
|
||||
let entry = entry.with_context(|| format!("Failed to read {}", root.display()))?;
|
||||
if !entry.file_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
let rel_from_root = entry
|
||||
.path()
|
||||
.strip_prefix(&root)
|
||||
.with_context(|| format!("Failed to relativize {}", entry.path().display()))?
|
||||
.to_path_buf();
|
||||
let rel_path = root_rel.join(rel_from_root);
|
||||
let key = rel_path_key(&rel_path);
|
||||
if !predicate(&key) {
|
||||
continue;
|
||||
}
|
||||
result.insert(key, timestamp_ms(entry.metadata()?.modified()?)?);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn should_walk_entry(path: &Path, skip_dirs: &BTreeSet<&str>) -> bool {
|
||||
path.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.is_none_or(|name| !skip_dirs.contains(name))
|
||||
}
|
||||
|
||||
fn gather_wasm_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
let markdown_parser_rust_dir = PathBuf::from("../packages/markdown_parser/rust");
|
||||
let mut inputs = collect_file_stats(
|
||||
project_root,
|
||||
&[
|
||||
PathBuf::from("../tools/ci/Cargo.toml"),
|
||||
PathBuf::from("../tools/ci/src/app_dev_server.rs"),
|
||||
PathBuf::from("../tools/ci/src/app_wasm.rs"),
|
||||
PathBuf::from("../tools/ci/src/common.rs"),
|
||||
PathBuf::from("../tools/ci/src/lib.rs"),
|
||||
PathBuf::from("../tools/ci/templates/libfluxcore_wrapper.js"),
|
||||
PathBuf::from("../tools/ci/templates/libfluxcore_wrapper.d.ts"),
|
||||
markdown_parser_rust_dir.join("Cargo.toml"),
|
||||
],
|
||||
)?;
|
||||
inputs.extend(collect_directory_stats(
|
||||
project_root,
|
||||
Path::new("rust/libfluxcore"),
|
||||
|path| !path.contains("/target/"),
|
||||
)?);
|
||||
inputs.extend(collect_directory_stats(
|
||||
project_root,
|
||||
&markdown_parser_rust_dir,
|
||||
|path| !path.contains("/target/"),
|
||||
)?);
|
||||
Ok(inputs)
|
||||
}
|
||||
|
||||
fn gather_color_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_file_stats(
|
||||
project_root,
|
||||
&[PathBuf::from("scripts/GenerateColorSystem.ts")],
|
||||
)
|
||||
}
|
||||
|
||||
fn gather_message_layout_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_file_stats(
|
||||
project_root,
|
||||
&[
|
||||
PathBuf::from("scripts/GenerateMessageLayoutCss.ts"),
|
||||
PathBuf::from("src/features/theme/layout/MessageLayoutSpec.ts"),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
fn gather_mask_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_file_stats(
|
||||
project_root,
|
||||
&[
|
||||
PathBuf::from("scripts/GenerateAvatarMasks.ts"),
|
||||
PathBuf::from("src/features/ui/constants/TypingConstants.ts"),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
fn gather_css_module_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_directory_stats(project_root, Path::new("src"), |path| {
|
||||
path.ends_with(".module.css")
|
||||
})
|
||||
}
|
||||
|
||||
fn gather_lingui_inputs(project_root: &Path) -> Result<BTreeMap<String, f64>> {
|
||||
collect_directory_stats(
|
||||
project_root,
|
||||
Path::new("src/features/i18n/locales"),
|
||||
|path| path.ends_with(".po"),
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_child(command: &str, args: &[&str], cwd: &Path) -> Result<Child> {
|
||||
println!("+ {}", display_command(command, args));
|
||||
Command::new(command)
|
||||
.args(args)
|
||||
.current_dir(cwd)
|
||||
.stdin(Stdio::inherit())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.with_context(|| format!("Failed to run {}", display_command(command, args)))
|
||||
}
|
||||
|
||||
async fn wait_for_child(
|
||||
command: &str,
|
||||
args: &[&str],
|
||||
child: &mut Child,
|
||||
shutdown: &mut watch::Receiver<bool>,
|
||||
) -> Result<std::process::ExitStatus> {
|
||||
tokio::select! {
|
||||
status = child.wait() => {
|
||||
status.with_context(|| format!("Failed to wait for {}", display_command(command, args)))
|
||||
}
|
||||
changed = shutdown.changed() => {
|
||||
let _ = changed;
|
||||
terminate_child(child).await;
|
||||
child.wait().await.with_context(|| format!("Failed to wait for {}", display_command(command, args)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn terminate_child(child: &mut Child) {
|
||||
if child.id().is_none() {
|
||||
return;
|
||||
}
|
||||
let _ = child.start_kill();
|
||||
let _ = timeout(Duration::from_secs(5), child.wait()).await;
|
||||
}
|
||||
|
||||
async fn listen_for_shutdown(shutdown_tx: watch::Sender<bool>) {
|
||||
let signal = wait_for_shutdown_signal().await;
|
||||
println!("\nReceived {signal}, shutting down fluxer app dev server...");
|
||||
let _ = shutdown_tx.send(true);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn wait_for_shutdown_signal() -> &'static str {
|
||||
let mut sigterm = match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
{
|
||||
Ok(signal) => signal,
|
||||
Err(_) => {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
return "SIGINT";
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => "SIGINT",
|
||||
_ = sigterm.recv() => "SIGTERM",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn wait_for_shutdown_signal() -> &'static str {
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
"SIGINT"
|
||||
}
|
||||
|
||||
fn rel_path_key(path: &Path) -> String {
|
||||
path.to_string_lossy().replace('\\', "/")
|
||||
}
|
||||
|
||||
fn timestamp_ms(timestamp: SystemTime) -> Result<f64> {
|
||||
Ok(timestamp
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.context("File timestamp predates UNIX epoch")?
|
||||
.as_secs_f64()
|
||||
* 1000.0)
|
||||
}
|
||||
|
||||
fn env_truthy(name: &str) -> bool {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.is_some_and(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true"))
|
||||
}
|
||||
|
||||
fn display_command(command: &str, args: &[&str]) -> String {
|
||||
std::iter::once(command.to_string())
|
||||
.chain(args.iter().map(|arg| quote_arg(arg)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
fn quote_arg(arg: &str) -> String {
|
||||
if arg
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.' | '/' | ':' | '='))
|
||||
{
|
||||
arg.to_string()
|
||||
} else {
|
||||
format!("{arg:?}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rel_path_keys_are_posix_like() {
|
||||
assert_eq!(
|
||||
rel_path_key(Path::new("scripts/GenerateColorSystem.ts")),
|
||||
"scripts/GenerateColorSystem.ts"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_command_quotes_globs() {
|
||||
assert_eq!(
|
||||
display_command("tcm", &["src", "--pattern", "**/*.module.css"]),
|
||||
"tcm src --pattern \"**/*.module.css\""
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{
|
||||
CALVER_SCHEME, CalverEnv, CommandSpec, S3UploadPlanItem, append_github_env,
|
||||
append_github_output, collect_files, path_to_s3_key, require_env, resolve_calver, run_command,
|
||||
runner_temp, s3_client, trim_option, upload_s3_plan_append_only,
|
||||
};
|
||||
use anyhow::{Context, Result, anyhow, ensure};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use chrono::Utc;
|
||||
use clap::{Args, ValueEnum};
|
||||
use serde_json::{Map, Value, json};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const DEFAULT_PUBLIC_ASSET_BASE_URL: &str = "https://fluxerstatic.com";
|
||||
const DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED: &str = "true";
|
||||
const DEFAULT_STATIC_BUCKET: &str = "fluxer-static";
|
||||
const DEFAULT_S3_ENDPOINT: &str = "https://ewr1.vultrobjects.com";
|
||||
const IMMUTABLE_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable";
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildAppProxyArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: AppProxyStep,
|
||||
#[arg(long)]
|
||||
build_version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
enum AppProxyStep {
|
||||
SetMetadata,
|
||||
PrepareDockerConfig,
|
||||
ConfigureGhcrAuth,
|
||||
BuildAndExtract,
|
||||
GenerateAssetManifest,
|
||||
UploadAssets,
|
||||
}
|
||||
|
||||
pub async fn run(args: BuildAppProxyArgs) -> Result<()> {
|
||||
match args.step {
|
||||
AppProxyStep::SetMetadata => set_metadata_step(args.build_version.as_deref()),
|
||||
AppProxyStep::PrepareDockerConfig => prepare_docker_config_step(),
|
||||
AppProxyStep::ConfigureGhcrAuth => configure_ghcr_auth_step(),
|
||||
AppProxyStep::BuildAndExtract => build_and_extract_step(),
|
||||
AppProxyStep::GenerateAssetManifest => generate_asset_manifest_step(),
|
||||
AppProxyStep::UploadAssets => upload_assets_step().await,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_metadata_step(build_version_arg: Option<&str>) -> Result<()> {
|
||||
let calver_env = CalverEnv {
|
||||
build_version: trim_option(build_version_arg.map(ToOwned::to_owned))
|
||||
.or_else(|| trim_option(env::var("BUILD_VERSION").ok())),
|
||||
fluxer_build_version: trim_option(env::var("FLUXER_BUILD_VERSION").ok()),
|
||||
fluxer_build_date: trim_option(env::var("FLUXER_BUILD_DATE").ok()),
|
||||
};
|
||||
let version = resolve_calver(&calver_env, Utc::now())?;
|
||||
append_github_output(&[
|
||||
("build_version", version.as_str()),
|
||||
("version", version.as_str()),
|
||||
("calver_scheme", CALVER_SCHEME),
|
||||
])
|
||||
}
|
||||
|
||||
fn prepare_docker_config_step() -> Result<()> {
|
||||
let docker_config = runner_temp().join("docker-config");
|
||||
fs::create_dir_all(&docker_config)
|
||||
.with_context(|| format!("Failed to create {}", docker_config.display()))?;
|
||||
append_github_env(&[("DOCKER_CONFIG", docker_config.to_string_lossy().as_ref())])
|
||||
}
|
||||
|
||||
fn configure_ghcr_auth_step() -> Result<()> {
|
||||
let docker_config = require_env("DOCKER_CONFIG")?;
|
||||
let username = require_env("GHCR_USERNAME")?;
|
||||
let token = require_env("GHCR_TOKEN")?;
|
||||
let path = PathBuf::from(docker_config).join("config.json");
|
||||
write_ghcr_auth_config(&path, &username, &token)
|
||||
}
|
||||
|
||||
fn write_ghcr_auth_config(path: &Path, username: &str, token: &str) -> Result<()> {
|
||||
let mut config = if path.exists() {
|
||||
serde_json::from_str::<Value>(
|
||||
&fs::read_to_string(path)
|
||||
.with_context(|| format!("Failed to read {}", path.display()))?,
|
||||
)
|
||||
.with_context(|| format!("Failed to parse {}", path.display()))?
|
||||
} else {
|
||||
Value::Object(Map::new())
|
||||
};
|
||||
|
||||
let root = config
|
||||
.as_object_mut()
|
||||
.ok_or_else(|| anyhow!("Docker config root must be a JSON object"))?;
|
||||
let auths = root
|
||||
.entry("auths")
|
||||
.or_insert_with(|| Value::Object(Map::new()))
|
||||
.as_object_mut()
|
||||
.ok_or_else(|| anyhow!("Docker config auths must be a JSON object"))?;
|
||||
auths.insert(
|
||||
"ghcr.io".to_string(),
|
||||
json!({ "auth": ghcr_auth_value(username, token) }),
|
||||
);
|
||||
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
fs::write(path, format!("{}\n", serde_json::to_string(&config)?))
|
||||
.with_context(|| format!("Failed to write {}", path.display()))
|
||||
}
|
||||
|
||||
fn ghcr_auth_value(username: &str, token: &str) -> String {
|
||||
BASE64.encode(format!("{username}:{token}"))
|
||||
}
|
||||
|
||||
fn build_and_extract_step() -> Result<()> {
|
||||
run_command(build_and_extract_command()?)
|
||||
}
|
||||
|
||||
fn build_and_extract_command() -> Result<CommandSpec> {
|
||||
let build_version = require_env("BUILD_VERSION")?;
|
||||
let public_asset_base_url = env::var("PUBLIC_ASSET_BASE_URL")
|
||||
.unwrap_or_else(|_| DEFAULT_PUBLIC_ASSET_BASE_URL.to_string());
|
||||
let image_repo = match env::var("IMAGE_REPO") {
|
||||
Ok(value) => value,
|
||||
Err(_) => format!("ghcr.io/{}/fluxer-app-proxy", ghcr_owner()?),
|
||||
};
|
||||
Ok(CommandSpec::new("docker")
|
||||
.args(["buildx", "bake", "-f", "fluxer_app_proxy/docker-bake.hcl"])
|
||||
.env("IMAGE_REPO", image_repo)
|
||||
.env("BUILD_VERSION", build_version)
|
||||
.env("PUBLIC_ASSET_BASE_URL", public_asset_base_url)
|
||||
.env(
|
||||
"FLUXER_APP_PROXY_TIME_FREEZE_ENABLED",
|
||||
env::var("FLUXER_APP_PROXY_TIME_FREEZE_ENABLED")
|
||||
.unwrap_or_else(|_| DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED.to_string()),
|
||||
)
|
||||
.env(
|
||||
"CACHE_FROM",
|
||||
env::var("CACHE_FROM")
|
||||
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy".to_string()),
|
||||
)
|
||||
.env(
|
||||
"CACHE_TO",
|
||||
env::var("CACHE_TO")
|
||||
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy,mode=max".to_string()),
|
||||
)
|
||||
.env(
|
||||
"DOCKER_BUILD_SUMMARY",
|
||||
env::var("DOCKER_BUILD_SUMMARY").unwrap_or_else(|_| "false".to_string()),
|
||||
)
|
||||
.env(
|
||||
"DOCKER_BUILD_RECORD_UPLOAD",
|
||||
env::var("DOCKER_BUILD_RECORD_UPLOAD").unwrap_or_else(|_| "false".to_string()),
|
||||
))
|
||||
}
|
||||
|
||||
fn ghcr_owner() -> Result<String> {
|
||||
for key in ["GHCR_OWNER", "GITHUB_REPOSITORY_OWNER", "OWNER"] {
|
||||
if let Ok(value) = env::var(key) {
|
||||
let value = value.trim();
|
||||
if !value.is_empty() {
|
||||
return Ok(value.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(repository) = env::var("GITHUB_REPOSITORY")
|
||||
&& let Some((owner, _)) = repository.split_once('/')
|
||||
{
|
||||
let owner = owner.trim();
|
||||
if !owner.is_empty() {
|
||||
return Ok(owner.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow!(
|
||||
"GHCR owner must be set with GHCR_OWNER, GITHUB_REPOSITORY_OWNER, OWNER, or GITHUB_REPOSITORY"
|
||||
))
|
||||
}
|
||||
|
||||
fn generate_asset_manifest_step() -> Result<()> {
|
||||
let dist = app_dist_dir();
|
||||
let manifest_path = dist.join("assets-manifest.txt");
|
||||
let assets = asset_manifest_entries(&dist)?;
|
||||
fs::write(&manifest_path, format!("{}\n", assets.join("\n")))
|
||||
.with_context(|| format!("Failed to write {}", manifest_path.display()))?;
|
||||
println!("=== asset manifest ===");
|
||||
for asset in &assets {
|
||||
println!("{asset}");
|
||||
}
|
||||
println!("total assets: {}", assets.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn app_dist_dir() -> PathBuf {
|
||||
env::var("APP_DIST_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from("app-dist-output/dist"))
|
||||
}
|
||||
|
||||
fn asset_manifest_entries(dist: &Path) -> Result<Vec<String>> {
|
||||
let assets_dir = dist.join("assets");
|
||||
ensure!(
|
||||
assets_dir.exists(),
|
||||
"App proxy assets directory is missing: {}",
|
||||
assets_dir.display()
|
||||
);
|
||||
let mut entries = collect_files(&assets_dir)?
|
||||
.into_iter()
|
||||
.filter(|path| !is_source_map_asset(path))
|
||||
.map(|path| {
|
||||
path.strip_prefix(dist)
|
||||
.with_context(|| format!("Failed to relativize {}", path.display()))
|
||||
.map(path_to_s3_key)
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
entries.sort();
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
fn is_source_map_asset(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(|ext| ext.to_str())
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("map"))
|
||||
}
|
||||
|
||||
async fn upload_assets_step() -> Result<()> {
|
||||
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
|
||||
let bucket = env::var("STATIC_BUCKET").unwrap_or_else(|_| DEFAULT_STATIC_BUCKET.to_string());
|
||||
let dist = app_dist_dir();
|
||||
let manifest_path = dist.join("assets-manifest.txt");
|
||||
let assets = read_asset_manifest(&manifest_path)?;
|
||||
ensure!(!assets.is_empty(), "{} is empty", manifest_path.display());
|
||||
|
||||
let plan = asset_upload_plan(&dist, &assets)?;
|
||||
let stats = upload_s3_plan_append_only(&client, &bucket, plan).await?;
|
||||
|
||||
println!("upload complete - {} assets", assets.len());
|
||||
println!(
|
||||
"append-only result - uploaded {}, skipped existing {}, repaired metadata {}",
|
||||
stats.uploaded, stats.skipped_existing, stats.metadata_repaired
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn asset_upload_plan(dist: &Path, assets: &[String]) -> Result<Vec<S3UploadPlanItem>> {
|
||||
assets
|
||||
.iter()
|
||||
.map(|asset| {
|
||||
let path = dist.join(asset);
|
||||
ensure!(
|
||||
path.is_file(),
|
||||
"Manifest asset is missing: {}",
|
||||
path.display()
|
||||
);
|
||||
Ok(S3UploadPlanItem::new(path, asset.clone())
|
||||
.with_detected_content_type()
|
||||
.with_cache_control(IMMUTABLE_ASSET_CACHE_CONTROL)
|
||||
.repair_existing_metadata())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn read_asset_manifest(path: &Path) -> Result<Vec<String>> {
|
||||
let manifest =
|
||||
fs::read_to_string(path).with_context(|| format!("Failed to read {}", path.display()))?;
|
||||
manifest
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(validate_manifest_asset)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn validate_manifest_asset(asset: &str) -> Result<String> {
|
||||
ensure!(
|
||||
asset.starts_with("assets/"),
|
||||
"Asset manifest entry must be under assets/: {asset}"
|
||||
);
|
||||
ensure!(
|
||||
!asset.contains("..") && !asset.starts_with('/') && !asset.contains('\\'),
|
||||
"Invalid asset manifest path: {asset}"
|
||||
);
|
||||
Ok(asset.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::common::parse_version_instant;
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use std::ffi::OsString;
|
||||
|
||||
fn dt(year: i32, month: u32, day: u32, hour: u32, minute: u32, second: u32) -> DateTime<Utc> {
|
||||
Utc.with_ymd_and_hms(year, month, day, hour, minute, second)
|
||||
.single()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn write_file(path: &Path, contents: &str) {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_calver_from_explicit_or_date_override() {
|
||||
let explicit = CalverEnv {
|
||||
build_version: Some("2026.520.1".to_string()),
|
||||
fluxer_build_version: Some("2026.521.2".to_string()),
|
||||
fluxer_build_date: Some("2026-05-22T03:04:05Z".to_string()),
|
||||
};
|
||||
assert_eq!(
|
||||
resolve_calver(&explicit, dt(2026, 1, 1, 0, 0, 0)).unwrap(),
|
||||
"2026.520.1"
|
||||
);
|
||||
|
||||
let generated = CalverEnv {
|
||||
fluxer_build_date: Some("2026-05-20T01:02:03Z".to_string()),
|
||||
..CalverEnv::default()
|
||||
};
|
||||
assert_eq!(
|
||||
resolve_calver(&generated, dt(2026, 1, 1, 0, 0, 0)).unwrap(),
|
||||
"2026.520.10203"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_calver_time() {
|
||||
assert_eq!(
|
||||
parse_version_instant("2026.520.246000")
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"Invalid build version date/time: 2026.520.246000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghcr_auth_config_merges_existing_auths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let config_path = temp.path().join("config.json");
|
||||
fs::write(
|
||||
&config_path,
|
||||
r#"{"auths":{"example.com":{"auth":"old"}},"currentContext":"builder"}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
write_ghcr_auth_config(&config_path, "octo", "secret").unwrap();
|
||||
|
||||
let config: Value =
|
||||
serde_json::from_str(&fs::read_to_string(config_path).unwrap()).unwrap();
|
||||
assert_eq!(config["auths"]["example.com"]["auth"], "old");
|
||||
assert_eq!(
|
||||
config["auths"]["ghcr.io"]["auth"],
|
||||
BASE64.encode("octo:secret")
|
||||
);
|
||||
assert_eq!(config["currentContext"], "builder");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ghcr_auth_config_rejects_non_object_roots_and_auths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let config_path = temp.path().join("config.json");
|
||||
fs::write(&config_path, "[]").unwrap();
|
||||
assert_eq!(
|
||||
write_ghcr_auth_config(&config_path, "octo", "secret")
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"Docker config root must be a JSON object"
|
||||
);
|
||||
|
||||
fs::write(&config_path, r#"{"auths":[]}"#).unwrap();
|
||||
assert_eq!(
|
||||
write_ghcr_auth_config(&config_path, "octo", "secret")
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
"Docker config auths must be a JSON object"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_command_sets_bake_environment() {
|
||||
let command = CommandSpec::new("docker")
|
||||
.args(["buildx", "bake", "-f", "fluxer_app_proxy/docker-bake.hcl"])
|
||||
.env("IMAGE_REPO", "ghcr.io/example/fluxer-app-proxy")
|
||||
.env("BUILD_VERSION", "2026.520.1")
|
||||
.env("PUBLIC_ASSET_BASE_URL", DEFAULT_PUBLIC_ASSET_BASE_URL)
|
||||
.env(
|
||||
"FLUXER_APP_PROXY_TIME_FREEZE_ENABLED",
|
||||
DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED,
|
||||
);
|
||||
|
||||
assert_eq!(command.program, OsString::from("docker"));
|
||||
assert_eq!(
|
||||
command.args,
|
||||
vec![
|
||||
OsString::from("buildx"),
|
||||
OsString::from("bake"),
|
||||
OsString::from("-f"),
|
||||
OsString::from("fluxer_app_proxy/docker-bake.hcl"),
|
||||
]
|
||||
);
|
||||
assert!(command.env.contains(&(
|
||||
OsString::from("BUILD_VERSION"),
|
||||
OsString::from("2026.520.1")
|
||||
)));
|
||||
assert!(command.env.contains(&(
|
||||
OsString::from("FLUXER_APP_PROXY_TIME_FREEZE_ENABLED"),
|
||||
OsString::from(DEFAULT_APP_PROXY_TIME_FREEZE_ENABLED)
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_manifest_entries_are_sorted_and_relative_to_dist() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
write_file(&dist.join("assets/z.js"), "z");
|
||||
write_file(&dist.join("assets/z.js.map"), "{}");
|
||||
write_file(&dist.join("assets/chunks/a.js"), "a");
|
||||
write_file(&dist.join("assets/chunks/a.js.map"), "{}");
|
||||
write_file(&dist.join("index.html"), "ignored");
|
||||
|
||||
assert_eq!(
|
||||
asset_manifest_entries(&dist).unwrap(),
|
||||
vec!["assets/chunks/a.js", "assets/z.js"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_manifest_entries_require_assets_directory() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
fs::create_dir_all(&dist).unwrap();
|
||||
|
||||
assert!(
|
||||
asset_manifest_entries(&dist)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("App proxy assets directory is missing")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_reader_trims_blank_lines_and_keeps_order() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let manifest = temp.path().join("assets-manifest.txt");
|
||||
fs::write(&manifest, "\n assets/b.js \n\nassets/a.js\n").unwrap();
|
||||
|
||||
assert_eq!(
|
||||
read_asset_manifest(&manifest).unwrap(),
|
||||
vec!["assets/b.js", "assets/a.js"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_upload_plan_preserves_manifest_keys() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
write_file(&dist.join("assets/a.js"), "a");
|
||||
write_file(&dist.join("assets/chunks/b.js"), "b");
|
||||
let assets = vec!["assets/a.js".to_string(), "assets/chunks/b.js".to_string()];
|
||||
|
||||
let plan = asset_upload_plan(&dist, &assets).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
plan.iter()
|
||||
.map(|item| item.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["assets/a.js", "assets/chunks/b.js"]
|
||||
);
|
||||
assert_eq!(plan[0].path, dist.join("assets/a.js"));
|
||||
assert_eq!(plan[1].path, dist.join("assets/chunks/b.js"));
|
||||
assert_eq!(
|
||||
plan[0].content_type.as_deref(),
|
||||
Some("application/javascript; charset=utf-8")
|
||||
);
|
||||
assert_eq!(
|
||||
plan[0].cache_control.as_deref(),
|
||||
Some(IMMUTABLE_ASSET_CACHE_CONTROL)
|
||||
);
|
||||
assert!(plan[0].repair_existing_metadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asset_upload_plan_rejects_manifest_entries_missing_on_disk() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let dist = temp.path().join("dist");
|
||||
fs::create_dir_all(&dist).unwrap();
|
||||
let assets = vec!["assets/missing.js".to_string()];
|
||||
|
||||
assert!(
|
||||
asset_upload_plan(&dist, &assets)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("Manifest asset is missing")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_reader_rejects_paths_outside_assets() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let manifest = temp.path().join("assets-manifest.txt");
|
||||
fs::write(&manifest, "assets/a.js\n../secret\n").unwrap();
|
||||
|
||||
assert!(read_asset_manifest(&manifest).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_reader_rejects_absolute_parent_and_backslash_paths() {
|
||||
for asset in ["/assets/a.js", "assets/../secret", r"assets\app.js"] {
|
||||
assert!(validate_manifest_asset(asset).is_err(), "{asset}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_to_s3_key_uses_forward_slashes() {
|
||||
assert_eq!(
|
||||
path_to_s3_key(Path::new("assets").join("chunks").join("a.js").as_path()),
|
||||
"assets/chunks/a.js"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{CommandSpec, command_succeeds, env_bool, output_text, run_command};
|
||||
use anyhow::{Context, Result, anyhow, ensure};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use clap::Args;
|
||||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tempfile::TempDir;
|
||||
|
||||
const LIBFLUXCORE_WASM_BINDGEN_VERSION: &str = "0.2.122";
|
||||
const LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES: u64 = 300 * 1024;
|
||||
const LIBFLUXCORE_WRAPPER_JS: &str = include_str!("../templates/libfluxcore_wrapper.js");
|
||||
const LIBFLUXCORE_WRAPPER_DTS: &str = include_str!("../templates/libfluxcore_wrapper.d.ts");
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildAppWasmArgs {
|
||||
#[arg(long)]
|
||||
app_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildMarkdownParserWasmArgs {
|
||||
#[arg(long)]
|
||||
app_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
pub fn run_build_app_wasm(args: BuildAppWasmArgs) -> Result<()> {
|
||||
let app_dir = args.app_dir.unwrap_or(resolve_app_dir()?);
|
||||
build_markdown_parser_wasm(&app_dir)?;
|
||||
build_libfluxcore_wasm(&app_dir)
|
||||
}
|
||||
|
||||
pub fn run_build_markdown_parser_wasm(args: BuildMarkdownParserWasmArgs) -> Result<()> {
|
||||
let app_dir = args.app_dir.unwrap_or(resolve_app_dir()?);
|
||||
build_markdown_parser_wasm(&app_dir)
|
||||
}
|
||||
|
||||
fn build_markdown_parser_wasm(app_dir: &Path) -> Result<()> {
|
||||
let rust_source_dir = app_dir.join("../packages/markdown_parser/rust");
|
||||
let bytes_path =
|
||||
app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts");
|
||||
let temp = TempDir::new().context("Failed to create source temp directory")?;
|
||||
let target_dir = temp.path().join("target");
|
||||
|
||||
run_command(
|
||||
CommandSpec::new("cargo")
|
||||
.args(["build", "--release", "--target", "wasm32-unknown-unknown"])
|
||||
.env("CARGO_TARGET_DIR", target_dir.to_string_lossy().as_ref())
|
||||
.current_dir(&rust_source_dir),
|
||||
)?;
|
||||
|
||||
let wasm_path = target_dir.join("wasm32-unknown-unknown/release/fluxer_markdown_parser.wasm");
|
||||
let wasm =
|
||||
fs::read(&wasm_path).with_context(|| format!("Failed to read {}", wasm_path.display()))?;
|
||||
let content = markdown_wasm_bytes_content(&wasm);
|
||||
if let Some(parent) = bytes_path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
fs::write(&bytes_path, content)
|
||||
.with_context(|| format!("Failed to write {}", bytes_path.display()))
|
||||
}
|
||||
|
||||
fn build_libfluxcore_wasm(app_dir: &Path) -> Result<()> {
|
||||
let rust_package_dir = app_dir.join("rust/libfluxcore");
|
||||
let out_dir = app_dir.join("pkgs/libfluxcore");
|
||||
let wasm_path = out_dir.join("libfluxcore_bg.wasm");
|
||||
let previous_wasm_size = file_size(&wasm_path)?;
|
||||
|
||||
fs::create_dir_all(&out_dir)
|
||||
.with_context(|| format!("Failed to create {}", out_dir.display()))?;
|
||||
|
||||
let mut build = CommandSpec::new("cargo")
|
||||
.args([
|
||||
"build",
|
||||
"--release",
|
||||
"--target",
|
||||
"wasm32-unknown-unknown",
|
||||
"--manifest-path",
|
||||
])
|
||||
.arg(rust_package_dir.join("Cargo.toml"))
|
||||
.current_dir(&rust_package_dir);
|
||||
|
||||
if env_bool("FLUXCORE_WASM_SIMD") {
|
||||
let rustflags = match env::var("RUSTFLAGS") {
|
||||
Ok(value) if !value.trim().is_empty() => format!("{value} -C target-feature=+simd128"),
|
||||
_ => "-C target-feature=+simd128".to_string(),
|
||||
};
|
||||
build = build.env("RUSTFLAGS", rustflags);
|
||||
}
|
||||
|
||||
run_command(build)?;
|
||||
let wasm_bindgen = ensure_wasm_bindgen_cli()?;
|
||||
let temp = TempDir::new().context("Failed to create libfluxcore wasm-bindgen temp dir")?;
|
||||
let bindgen_dir = temp.path().join("bindgen");
|
||||
fs::create_dir_all(&bindgen_dir)
|
||||
.with_context(|| format!("Failed to create {}", bindgen_dir.display()))?;
|
||||
|
||||
run_command(
|
||||
CommandSpec::new(wasm_bindgen)
|
||||
.args(["--target", "web", "--out-dir"])
|
||||
.arg(&bindgen_dir)
|
||||
.args(["--out-name", "libfluxcore"])
|
||||
.arg(rust_package_dir.join("target/wasm32-unknown-unknown/release/libfluxcore.wasm"))
|
||||
.current_dir(&rust_package_dir),
|
||||
)?;
|
||||
|
||||
let bindgen_js_path = bindgen_dir.join("libfluxcore.js");
|
||||
let bindgen_dts_path = bindgen_dir.join("libfluxcore.d.ts");
|
||||
let bindgen_wasm_path = bindgen_dir.join("libfluxcore_bg.wasm");
|
||||
let bindgen_wasm_dts_path = bindgen_dir.join("libfluxcore_bg.wasm.d.ts");
|
||||
|
||||
write_with_spdx(
|
||||
&out_dir.join("libfluxcore_bindgen.js"),
|
||||
&patch_libfluxcore_bindgen_js(
|
||||
&fs::read_to_string(&bindgen_js_path)
|
||||
.with_context(|| format!("Failed to read {}", bindgen_js_path.display()))?,
|
||||
)?,
|
||||
)?;
|
||||
write_with_spdx(
|
||||
&out_dir.join("libfluxcore_bindgen.d.ts"),
|
||||
&patch_libfluxcore_bindgen_dts(
|
||||
&fs::read_to_string(&bindgen_dts_path)
|
||||
.with_context(|| format!("Failed to read {}", bindgen_dts_path.display()))?,
|
||||
)?,
|
||||
)?;
|
||||
fs::copy(&bindgen_wasm_path, &wasm_path).with_context(|| {
|
||||
format!(
|
||||
"Failed to copy {} to {}",
|
||||
bindgen_wasm_path.display(),
|
||||
wasm_path.display()
|
||||
)
|
||||
})?;
|
||||
fs::copy(
|
||||
&bindgen_wasm_dts_path,
|
||||
out_dir.join("libfluxcore_bg.wasm.d.ts"),
|
||||
)
|
||||
.with_context(|| format!("Failed to copy {}", bindgen_wasm_dts_path.display()))?;
|
||||
|
||||
fs::write(
|
||||
out_dir.join("libfluxcore.js"),
|
||||
libfluxcore_index_js_content(),
|
||||
)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to write {}",
|
||||
out_dir.join("libfluxcore.js").display()
|
||||
)
|
||||
})?;
|
||||
fs::write(
|
||||
out_dir.join("libfluxcore.d.ts"),
|
||||
libfluxcore_index_dts_content(),
|
||||
)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to write {}",
|
||||
out_dir.join("libfluxcore.d.ts").display()
|
||||
)
|
||||
})?;
|
||||
fs::write(
|
||||
out_dir.join("package.json"),
|
||||
libfluxcore_package_json_content(),
|
||||
)
|
||||
.with_context(|| format!("Failed to write {}", out_dir.join("package.json").display()))?;
|
||||
fs::write(out_dir.join("README.md"), libfluxcore_readme_content())
|
||||
.with_context(|| format!("Failed to write {}", out_dir.join("README.md").display()))?;
|
||||
|
||||
let wasm_size = file_size(&wasm_path)?
|
||||
.ok_or_else(|| anyhow!("libfluxcore build did not emit {}", wasm_path.display()))?;
|
||||
ensure!(
|
||||
wasm_size <= LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES,
|
||||
"libfluxcore_bg.wasm is {}, over the {} budget",
|
||||
format_bytes(wasm_size),
|
||||
format_bytes(LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES)
|
||||
);
|
||||
|
||||
let size_comparison = match previous_wasm_size {
|
||||
Some(previous) => format!("{} -> {}", format_bytes(previous), format_bytes(wasm_size)),
|
||||
None => "no previous artifact".to_string(),
|
||||
};
|
||||
println!(
|
||||
"libfluxcore_bg.wasm size: {size_comparison} (budget {})",
|
||||
format_bytes(LIBFLUXCORE_WASM_SIZE_BUDGET_BYTES)
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn patch_libfluxcore_bindgen_js(content: &str) -> Result<String> {
|
||||
const MARKER: &str = "\nasync function __wbg_load(module, imports) {";
|
||||
const RESET_EXPORT: &str = r#"
|
||||
export function __resetLibfluxcoreWasmForMemoryPressure() {
|
||||
wasmModule = undefined;
|
||||
wasmInstance = undefined;
|
||||
wasm = undefined;
|
||||
cachedDataViewMemory0 = null;
|
||||
cachedUint8ArrayMemory0 = null;
|
||||
heap = new Array(1024).fill(undefined);
|
||||
heap.push(undefined, null, true, false);
|
||||
heap_next = heap.length;
|
||||
numBytesDecoded = 0;
|
||||
}
|
||||
"#;
|
||||
ensure!(
|
||||
content.contains(MARKER),
|
||||
"libfluxcore wasm-bindgen JS output did not contain reset insertion marker"
|
||||
);
|
||||
Ok(content.replacen(MARKER, &format!("{RESET_EXPORT}{MARKER}"), 1))
|
||||
}
|
||||
|
||||
fn patch_libfluxcore_bindgen_dts(content: &str) -> Result<String> {
|
||||
const MARKER: &str = "\nexport type InitInput";
|
||||
const RESET_EXPORT: &str =
|
||||
"\nexport function __resetLibfluxcoreWasmForMemoryPressure(): void;\n";
|
||||
ensure!(
|
||||
content.contains(MARKER),
|
||||
"libfluxcore wasm-bindgen DTS output did not contain reset insertion marker"
|
||||
);
|
||||
Ok(content.replacen(MARKER, &format!("{RESET_EXPORT}{MARKER}"), 1))
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_app_dir() -> Result<PathBuf> {
|
||||
let cwd = env::current_dir().context("Failed to resolve current directory")?;
|
||||
if cwd.file_name().and_then(|value| value.to_str()) == Some("fluxer_app") {
|
||||
return Ok(cwd);
|
||||
}
|
||||
if cwd.join("fluxer_app").is_dir() {
|
||||
return Ok(cwd.join("fluxer_app"));
|
||||
}
|
||||
Err(anyhow!(
|
||||
"Could not resolve fluxer_app directory from {}",
|
||||
cwd.display()
|
||||
))
|
||||
}
|
||||
|
||||
fn ensure_wasm_bindgen_cli() -> Result<OsString> {
|
||||
let expected = format!("wasm-bindgen {LIBFLUXCORE_WASM_BINDGEN_VERSION}");
|
||||
if command_succeeds(CommandSpec::new("wasm-bindgen").arg("--version")) {
|
||||
let version = output_text(CommandSpec::new("wasm-bindgen").arg("--version"))?;
|
||||
if version.trim() == expected {
|
||||
return Ok("wasm-bindgen".into());
|
||||
}
|
||||
}
|
||||
|
||||
run_command(CommandSpec::new("cargo").args([
|
||||
"install",
|
||||
"wasm-bindgen-cli",
|
||||
"--version",
|
||||
LIBFLUXCORE_WASM_BINDGEN_VERSION,
|
||||
"--locked",
|
||||
"--force",
|
||||
]))?;
|
||||
Ok("wasm-bindgen".into())
|
||||
}
|
||||
|
||||
fn file_size(path: &Path) -> Result<Option<u64>> {
|
||||
match fs::metadata(path) {
|
||||
Ok(metadata) => Ok(Some(metadata.len())),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error).with_context(|| format!("Failed to stat {}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_bytes(bytes: u64) -> String {
|
||||
format!("{bytes} B")
|
||||
}
|
||||
|
||||
fn write_with_spdx(path: &Path, content: &str) -> Result<()> {
|
||||
fs::write(
|
||||
path,
|
||||
format!("// SPDX-License-Identifier: AGPL-3.0-or-later\n{content}"),
|
||||
)
|
||||
.with_context(|| format!("Failed to write {}", path.display()))
|
||||
}
|
||||
|
||||
fn libfluxcore_index_js_content() -> String {
|
||||
format!(
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
import {{crop_rotate_rgba_raw}} from './libfluxcore_bindgen.js';\n\
|
||||
{LIBFLUXCORE_WRAPPER_JS}\n\
|
||||
export * from './libfluxcore_bindgen.js';\n\
|
||||
export {{default}} from './libfluxcore_bindgen.js';\n"
|
||||
)
|
||||
}
|
||||
|
||||
fn libfluxcore_index_dts_content() -> String {
|
||||
format!(
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
export * from './libfluxcore_bindgen.js';\n\
|
||||
export {{default}} from './libfluxcore_bindgen.js';\n\n\
|
||||
{LIBFLUXCORE_WRAPPER_DTS}"
|
||||
)
|
||||
}
|
||||
|
||||
fn libfluxcore_package_json_content() -> String {
|
||||
let manifest = serde_json::json!({
|
||||
"name": "libfluxcore",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"version": "0.0.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"sideEffects": false,
|
||||
"files": [
|
||||
"libfluxcore.js",
|
||||
"libfluxcore.d.ts",
|
||||
"libfluxcore_bindgen.js",
|
||||
"libfluxcore_bindgen.d.ts",
|
||||
"libfluxcore_bg.wasm",
|
||||
"libfluxcore_bg.wasm.d.ts",
|
||||
"README.md"
|
||||
],
|
||||
"main": "libfluxcore.js",
|
||||
"module": "libfluxcore.js",
|
||||
"types": "libfluxcore.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./libfluxcore.d.ts",
|
||||
"default": "./libfluxcore.js"
|
||||
},
|
||||
"./libfluxcore_bg.wasm": "./libfluxcore_bg.wasm"
|
||||
}
|
||||
});
|
||||
format!("{manifest:#}\n")
|
||||
}
|
||||
|
||||
fn libfluxcore_readme_content() -> &'static str {
|
||||
"<!-- SPDX-License-Identifier: AGPL-3.0-or-later -->\n\
|
||||
# libfluxcore\n\n\
|
||||
Rust WebAssembly helpers and JavaScript codec wrappers for Fluxer media processing.\n"
|
||||
}
|
||||
|
||||
fn markdown_wasm_bytes_content(wasm: &[u8]) -> String {
|
||||
format!(
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
export const MARKDOWN_PARSER_WASM_BASE64 =\n\
|
||||
\t'{}';\n",
|
||||
BASE64.encode(wasm)
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn markdown_wasm_bytes_content_matches_legacy_node_output() {
|
||||
assert_eq!(
|
||||
markdown_wasm_bytes_content(b"hello"),
|
||||
"// SPDX-License-Identifier: AGPL-3.0-or-later\n\n\
|
||||
export const MARKDOWN_PARSER_WASM_BASE64 =\n\
|
||||
\t'aGVsbG8=';\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libfluxcore_index_reexports_bindgen_module() {
|
||||
let content = libfluxcore_index_js_content();
|
||||
assert!(content.contains("import {crop_rotate_rgba_raw} from './libfluxcore_bindgen.js';"));
|
||||
assert!(content.contains("export * from './libfluxcore_bindgen.js';"));
|
||||
assert!(content.contains("export function crop_rotate_rgba("));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libfluxcore_bindgen_js_reset_hook_is_inserted() {
|
||||
let content =
|
||||
"function __wbg_finalize_init() {}\nasync function __wbg_load(module, imports) {}";
|
||||
let patched = patch_libfluxcore_bindgen_js(content).expect("patch should succeed");
|
||||
assert!(patched.contains("export function __resetLibfluxcoreWasmForMemoryPressure()"));
|
||||
assert!(patched.contains("wasm = undefined;"));
|
||||
assert!(patched.contains("async function __wbg_load(module, imports) {}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn libfluxcore_bindgen_dts_reset_hook_is_inserted() {
|
||||
let content = "export function is_animated_image(input: Uint8Array): boolean;\nexport type InitInput = RequestInfo;";
|
||||
let patched = patch_libfluxcore_bindgen_dts(content).expect("patch should succeed");
|
||||
assert!(
|
||||
patched.contains("export function __resetLibfluxcoreWasmForMemoryPressure(): void;")
|
||||
);
|
||||
assert!(patched.contains("export type InitInput = RequestInfo;"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{
|
||||
CALVER_SCHEME, CalverEnv, append_github_env, append_github_output, parse_version_instant,
|
||||
resolve_calver, trim_option,
|
||||
};
|
||||
use anyhow::Result;
|
||||
use chrono::{Datelike, Timelike, Utc};
|
||||
use clap::Args;
|
||||
use std::env;
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct ResolveCalverArgs {
|
||||
#[arg(long)]
|
||||
github_output: bool,
|
||||
#[arg(long)]
|
||||
github_env: bool,
|
||||
#[arg(long, default_value = "BUILD_VERSION")]
|
||||
env_name: String,
|
||||
}
|
||||
|
||||
pub fn run(args: ResolveCalverArgs) -> Result<()> {
|
||||
let resolved = resolve_calver_from_env()?;
|
||||
if args.github_output {
|
||||
let output = calver_outputs(&resolved)?;
|
||||
append_github_output(&[
|
||||
("version", output.version.as_str()),
|
||||
("build_version", output.version.as_str()),
|
||||
("time", output.time.as_str()),
|
||||
("micro", output.micro.as_str()),
|
||||
("patch", output.micro.as_str()),
|
||||
("date", output.date.as_str()),
|
||||
("year", output.year.as_str()),
|
||||
("month", output.month.as_str()),
|
||||
("day", output.day.as_str()),
|
||||
("month_day", output.month_day.as_str()),
|
||||
("calver_scheme", CALVER_SCHEME),
|
||||
])?;
|
||||
}
|
||||
if args.github_env {
|
||||
append_github_env(&[(args.env_name.as_str(), resolved.as_str())])?;
|
||||
}
|
||||
println!("{resolved}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resolve_calver_from_env() -> Result<String> {
|
||||
resolve_calver(
|
||||
&CalverEnv {
|
||||
build_version: trim_option(env::var("BUILD_VERSION").ok()),
|
||||
fluxer_build_version: trim_option(env::var("FLUXER_BUILD_VERSION").ok()),
|
||||
fluxer_build_date: trim_option(env::var("FLUXER_BUILD_DATE").ok()),
|
||||
},
|
||||
Utc::now(),
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct CalverOutputs {
|
||||
version: String,
|
||||
time: String,
|
||||
micro: String,
|
||||
date: String,
|
||||
year: String,
|
||||
month: String,
|
||||
day: String,
|
||||
month_day: String,
|
||||
}
|
||||
|
||||
fn calver_outputs(version: &str) -> Result<CalverOutputs> {
|
||||
let instant = parse_version_instant(version)?;
|
||||
let time = format!(
|
||||
"{:02}{:02}{:02}",
|
||||
instant.hour(),
|
||||
instant.minute(),
|
||||
instant.second()
|
||||
);
|
||||
let micro = time
|
||||
.parse::<u32>()
|
||||
.expect("HHMMSS time segment should parse")
|
||||
.to_string();
|
||||
Ok(CalverOutputs {
|
||||
version: version.to_string(),
|
||||
time,
|
||||
micro,
|
||||
date: format!(
|
||||
"{:04}{:02}{:02}",
|
||||
instant.year(),
|
||||
instant.month(),
|
||||
instant.day()
|
||||
),
|
||||
year: instant.year().to_string(),
|
||||
month: instant.month().to_string(),
|
||||
day: format!("{:02}", instant.day()),
|
||||
month_day: format!("{}{:02}", instant.month(), instant.day()),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::common::CalverEnv;
|
||||
use chrono::{TimeZone, Utc};
|
||||
|
||||
#[test]
|
||||
fn calver_outputs_match_legacy_shell_fields() {
|
||||
assert_eq!(
|
||||
calver_outputs("2026.520.10203").unwrap(),
|
||||
CalverOutputs {
|
||||
version: "2026.520.10203".to_string(),
|
||||
time: "010203".to_string(),
|
||||
micro: "10203".to_string(),
|
||||
date: "20260520".to_string(),
|
||||
year: "2026".to_string(),
|
||||
month: "5".to_string(),
|
||||
day: "20".to_string(),
|
||||
month_day: "520".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calver_date_only_override_matches_legacy_shell() {
|
||||
let version = resolve_calver(
|
||||
&CalverEnv {
|
||||
fluxer_build_date: Some("2026-01-09".to_string()),
|
||||
..CalverEnv::default()
|
||||
},
|
||||
Utc.with_ymd_and_hms(2026, 5, 20, 1, 2, 3).single().unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(version, "2026.109.0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calver_rejects_invalid_time() {
|
||||
assert_eq!(
|
||||
calver_outputs("2026.520.246000").unwrap_err().to_string(),
|
||||
"Invalid build version date/time: 2026.520.246000"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{CommandSpec, run_command};
|
||||
use crate::desktop::write_build_channel_file;
|
||||
use crate::gateway::{GatewayStep, run_gateway_step};
|
||||
use anyhow::{Context, Result};
|
||||
use clap::{Args, ValueEnum};
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct CiArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: CiStep,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
enum CiStep {
|
||||
InstallDependencies,
|
||||
Typecheck,
|
||||
Test,
|
||||
Knip,
|
||||
GatewayFmt,
|
||||
GatewayCompile,
|
||||
GatewayDialyzer,
|
||||
GatewayEunit,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct CiScriptsArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: CiScriptsStep,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
enum CiScriptsStep {
|
||||
Sync,
|
||||
Test,
|
||||
}
|
||||
|
||||
pub async fn run_ci(args: CiArgs) -> Result<()> {
|
||||
let root = repo_root()?;
|
||||
match args.step {
|
||||
CiStep::InstallDependencies => run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["install", "--frozen-lockfile"])
|
||||
.current_dir(root),
|
||||
),
|
||||
CiStep::Typecheck => {
|
||||
ensure_desktop_build_channel_file(&root)?;
|
||||
run_generators(&root, true)?;
|
||||
run_app_test_artifact_generators(&root)?;
|
||||
run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["-r", "--if-present", "typecheck"])
|
||||
.current_dir(root),
|
||||
)
|
||||
}
|
||||
CiStep::Test => {
|
||||
run_generators(&root, false)?;
|
||||
run_app_test_artifact_generators(&root)?;
|
||||
run_workspace_tests(&root)?;
|
||||
run_command(with_test_env(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["--filter", "fluxer_api", "test"])
|
||||
.current_dir(root),
|
||||
))
|
||||
}
|
||||
CiStep::Knip => {
|
||||
run_app_test_artifact_generators(&root)?;
|
||||
ensure_desktop_build_channel_file(&root)?;
|
||||
run_fluxer_app_script(&root, "i18n:compile")?;
|
||||
run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["exec", "knip"])
|
||||
.current_dir(root),
|
||||
)
|
||||
}
|
||||
CiStep::GatewayFmt => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::FmtCheck, "test")
|
||||
}
|
||||
CiStep::GatewayCompile => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Compile, "test")
|
||||
}
|
||||
CiStep::GatewayDialyzer => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Dialyzer, "test")
|
||||
}
|
||||
CiStep::GatewayEunit => {
|
||||
run_gateway_step(&root.join("fluxer_gateway"), GatewayStep::Eunit, "test")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_desktop_build_channel_file(root: &Path) -> Result<()> {
|
||||
let channel = env::var("BUILD_CHANNEL").unwrap_or_else(|_| "stable".to_string());
|
||||
write_build_channel_file(&root.join("fluxer_desktop"), &channel)
|
||||
}
|
||||
|
||||
fn run_app_test_artifact_generators(root: &Path) -> Result<()> {
|
||||
run_fluxer_app_script(root, "wasm:codegen")?;
|
||||
run_fluxer_app_script(root, "generate:masks")
|
||||
}
|
||||
|
||||
fn run_fluxer_app_script(root: &Path, script: &str) -> Result<()> {
|
||||
run_command(
|
||||
CommandSpec::new("pnpm")
|
||||
.args(["--filter", "fluxer_app", script])
|
||||
.current_dir(root),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn run_ci_scripts(args: CiScriptsArgs) -> Result<()> {
|
||||
let root = repo_root()?;
|
||||
match args.step {
|
||||
CiScriptsStep::Sync => run_command(
|
||||
CommandSpec::new("cargo")
|
||||
.args([
|
||||
"fetch",
|
||||
"--locked",
|
||||
"--manifest-path",
|
||||
"tools/ci/Cargo.toml",
|
||||
])
|
||||
.current_dir(root),
|
||||
),
|
||||
CiScriptsStep::Test => run_command(
|
||||
CommandSpec::new("cargo")
|
||||
.args(["test", "--locked", "--manifest-path", "tools/ci/Cargo.toml"])
|
||||
.current_dir(root),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_generators(root: &Path, for_typecheck: bool) -> Result<()> {
|
||||
for command in generator_commands(for_typecheck) {
|
||||
run_command(command.current_dir(root))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generator_commands(for_typecheck: bool) -> Vec<CommandSpec> {
|
||||
let mut commands = vec![
|
||||
CommandSpec::new("pnpm").args(["--filter", "@fluxer/config", "generate"]),
|
||||
CommandSpec::new("pnpm").args(["--filter", "@fluxer/schema", "generate"]),
|
||||
];
|
||||
if for_typecheck {
|
||||
commands.push(CommandSpec::new("pnpm").args([
|
||||
"--filter",
|
||||
"@fluxer/i18n",
|
||||
"generate:types",
|
||||
]));
|
||||
}
|
||||
commands.push(CommandSpec::new("pnpm").args(["--filter", "fluxer_app", "i18n:compile"]));
|
||||
commands
|
||||
}
|
||||
|
||||
fn run_workspace_tests(root: &Path) -> Result<()> {
|
||||
let workspace_concurrency =
|
||||
env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").unwrap_or_else(|_| "2".to_string());
|
||||
run_command(with_test_env(
|
||||
CommandSpec::new("pnpm")
|
||||
.args([
|
||||
"-r",
|
||||
&format!("--workspace-concurrency={workspace_concurrency}"),
|
||||
"--filter",
|
||||
"!fluxer_api",
|
||||
"--filter",
|
||||
"!fluxer",
|
||||
"--if-present",
|
||||
"test",
|
||||
])
|
||||
.current_dir(root),
|
||||
))
|
||||
}
|
||||
|
||||
fn with_test_env(spec: CommandSpec) -> CommandSpec {
|
||||
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
|
||||
let api_workers = env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| "2".to_string());
|
||||
spec.env("FLUXER_NATS_URL", &nats_url)
|
||||
.env(
|
||||
"FLUXER_NATS_CORE_URL",
|
||||
env::var("FLUXER_NATS_CORE_URL").unwrap_or_else(|_| nats_url.clone()),
|
||||
)
|
||||
.env(
|
||||
"FLUXER_NATS_JETSTREAM_URL",
|
||||
env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone()),
|
||||
)
|
||||
.env("API_TEST_MAX_WORKERS", &api_workers)
|
||||
.env(
|
||||
"API_TEST_MAX_CONCURRENCY",
|
||||
env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers),
|
||||
)
|
||||
}
|
||||
|
||||
fn default_test_nats_url() -> String {
|
||||
if Path::new("/.dockerenv").exists() {
|
||||
"nats://nats:4222".to_string()
|
||||
} else {
|
||||
"nats://127.0.0.1:4222".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_root() -> Result<PathBuf> {
|
||||
env::var("GITHUB_WORKSPACE")
|
||||
.map(PathBuf::from)
|
||||
.or_else(|_| env::current_dir())
|
||||
.context("Failed to resolve repository root")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::ffi::OsString;
|
||||
|
||||
#[test]
|
||||
fn generator_commands_include_i18n_types_only_for_typecheck() {
|
||||
let typecheck = generator_commands(true)
|
||||
.into_iter()
|
||||
.map(|command| command.args)
|
||||
.collect::<Vec<_>>();
|
||||
let test = generator_commands(false)
|
||||
.into_iter()
|
||||
.map(|command| command.args)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert!(typecheck.contains(&vec![
|
||||
OsString::from("--filter"),
|
||||
OsString::from("@fluxer/i18n"),
|
||||
OsString::from("generate:types"),
|
||||
]));
|
||||
assert!(!test.contains(&vec![
|
||||
OsString::from("--filter"),
|
||||
OsString::from("@fluxer/i18n"),
|
||||
OsString::from("generate:types"),
|
||||
]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_test_env_sets_all_nats_urls_and_concurrency() {
|
||||
let spec = with_test_env(CommandSpec::new("pnpm"));
|
||||
let env = spec
|
||||
.env
|
||||
.into_iter()
|
||||
.collect::<std::collections::BTreeMap<_, _>>();
|
||||
let default_nats_url = OsString::from(default_test_nats_url());
|
||||
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("FLUXER_NATS_URL")),
|
||||
Some(&default_nats_url)
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("FLUXER_NATS_CORE_URL")),
|
||||
Some(&default_nats_url)
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("FLUXER_NATS_JETSTREAM_URL")),
|
||||
Some(&default_nats_url)
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("API_TEST_MAX_WORKERS")),
|
||||
Some(&OsString::from("2"))
|
||||
);
|
||||
assert_eq!(
|
||||
env.get(&OsString::from("API_TEST_MAX_CONCURRENCY")),
|
||||
Some(&OsString::from("2"))
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Serialize;
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
pub(crate) fn remove_file_if_exists(path: &Path) -> Result<()> {
|
||||
match fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error).with_context(|| format!("Failed to remove {}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn remove_dir_if_exists(path: &Path) -> Result<()> {
|
||||
match fs::remove_dir_all(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error).with_context(|| format!("Failed to remove {}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn write_json_pretty<T: Serialize + ?Sized>(path: &Path, value: &T) -> Result<()> {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
let mut bytes = serde_json::to_vec_pretty(value)?;
|
||||
bytes.push(b'\n');
|
||||
fs::write(path, bytes).with_context(|| format!("Failed to write {}", path.display()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn remove_file_if_exists_removes_files_and_ignores_missing_paths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("artifact.txt");
|
||||
fs::write(&path, "artifact").unwrap();
|
||||
|
||||
remove_file_if_exists(&path).unwrap();
|
||||
remove_file_if_exists(&path).unwrap();
|
||||
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_dir_if_exists_removes_directories_and_ignores_missing_paths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("dist");
|
||||
fs::create_dir_all(path.join("nested")).unwrap();
|
||||
fs::write(path.join("nested").join("artifact.txt"), "artifact").unwrap();
|
||||
|
||||
remove_dir_if_exists(&path).unwrap();
|
||||
remove_dir_if_exists(&path).unwrap();
|
||||
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_json_pretty_creates_parent_directories_and_writes_pretty_json() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("nested").join("manifest.json");
|
||||
|
||||
write_json_pretty(&path, &json!({ "name": "fluxer", "version": 1 })).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
fs::read_to_string(path).unwrap(),
|
||||
"{\n \"name\": \"fluxer\",\n \"version\": 1\n}\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{CommandSpec, command_succeeds, output_text, run_command};
|
||||
use anyhow::{Context, Result, anyhow, bail, ensure};
|
||||
use clap::{Args, ValueEnum};
|
||||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const GATEWAY_NIF_CRATES: &[&str] = &["push_markdown_plaintext_nif", "guild_member_list_oset_nif"];
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct BuildGatewayNifsArgs {
|
||||
#[arg(long)]
|
||||
gateway_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct GatewayArgs {
|
||||
#[arg(long, value_enum)]
|
||||
step: GatewayStep,
|
||||
#[arg(long, default_value = "test")]
|
||||
eqwalizer_profile: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
pub(crate) enum GatewayStep {
|
||||
Fmt,
|
||||
FmtCheck,
|
||||
Lint,
|
||||
Compile,
|
||||
ProdCompile,
|
||||
Dialyzer,
|
||||
Eqwalizer,
|
||||
Typecheck,
|
||||
Eunit,
|
||||
Bench,
|
||||
AllChecks,
|
||||
Clean,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum NifBuildProfile {
|
||||
Debug,
|
||||
Release,
|
||||
}
|
||||
|
||||
impl NifBuildProfile {
|
||||
fn from_env_value(value: Option<&str>) -> Self {
|
||||
match value {
|
||||
Some("release") | None => Self::Release,
|
||||
Some(_) => Self::Debug,
|
||||
}
|
||||
}
|
||||
|
||||
fn target_dir_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Debug => "debug",
|
||||
Self::Release => "release",
|
||||
}
|
||||
}
|
||||
|
||||
fn cargo_args(self) -> Vec<OsString> {
|
||||
let mut args = vec![OsString::from("build")];
|
||||
if self == Self::Release {
|
||||
args.push(OsString::from("--release"));
|
||||
}
|
||||
args
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct GatewayNifBuild {
|
||||
crate_name: String,
|
||||
native_dir: PathBuf,
|
||||
cargo_args: Vec<OsString>,
|
||||
artifact_path: PathBuf,
|
||||
output_path: PathBuf,
|
||||
}
|
||||
|
||||
pub fn run_build_gateway_nifs(args: BuildGatewayNifsArgs) -> Result<()> {
|
||||
let gateway_dir = args
|
||||
.gateway_dir
|
||||
.map(Ok)
|
||||
.unwrap_or_else(resolve_gateway_dir)?;
|
||||
build_gateway_nifs(&gateway_dir)
|
||||
}
|
||||
|
||||
pub fn run_gateway(args: GatewayArgs) -> Result<()> {
|
||||
let gateway_dir = resolve_gateway_dir()?;
|
||||
run_gateway_step(&gateway_dir, args.step, &args.eqwalizer_profile)
|
||||
}
|
||||
|
||||
pub(crate) fn run_gateway_step(
|
||||
gateway_dir: &Path,
|
||||
step: GatewayStep,
|
||||
eqwalizer_profile: &str,
|
||||
) -> Result<()> {
|
||||
match step {
|
||||
GatewayStep::Fmt => run_rebar(gateway_dir, ["fmt"]),
|
||||
GatewayStep::FmtCheck => run_gateway_fmt_check(gateway_dir),
|
||||
GatewayStep::Lint => run_rebar(gateway_dir, ["lint"]),
|
||||
GatewayStep::Compile => run_rebar(gateway_dir, ["compile"]),
|
||||
GatewayStep::ProdCompile => {
|
||||
run_rebar(gateway_dir, ["as", "prod", "clean", "-a"])?;
|
||||
run_rebar(gateway_dir, ["as", "prod", "compile"])
|
||||
}
|
||||
GatewayStep::Dialyzer => run_rebar(gateway_dir, ["dialyzer"]),
|
||||
GatewayStep::Eqwalizer => run_eqwalizer(gateway_dir, eqwalizer_profile),
|
||||
GatewayStep::Typecheck => run_eqwalizer(gateway_dir, eqwalizer_profile),
|
||||
GatewayStep::Eunit => run_rebar(gateway_dir, ["as", "test", "eunit"]),
|
||||
GatewayStep::Bench => run_rebar(gateway_dir, ["eunit", "--module=guild_member_list_bench"]),
|
||||
GatewayStep::AllChecks => {
|
||||
run_gateway_check_step("Step 1/5: Format check (erlfmt)", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::FmtCheck, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 2/5: Lint (elvis)", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Lint, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 3/5: Compile", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Compile, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 4/5: Type check", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Typecheck, eqwalizer_profile)
|
||||
})?;
|
||||
run_gateway_check_step("Step 5/5: Unit tests (eunit)", || {
|
||||
run_gateway_step(gateway_dir, GatewayStep::Eunit, eqwalizer_profile)
|
||||
})?;
|
||||
println!("All gateway checks passed.");
|
||||
Ok(())
|
||||
}
|
||||
GatewayStep::Clean => {
|
||||
run_rebar(gateway_dir, ["clean", "--all"])?;
|
||||
let plugins_dir = gateway_dir.join("_build/default/plugins");
|
||||
if plugins_dir.exists() {
|
||||
fs::remove_dir_all(&plugins_dir)
|
||||
.with_context(|| format!("Failed to remove {}", plugins_dir.display()))?;
|
||||
}
|
||||
println!("Cleaned gateway build outputs.");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_gateway_nifs(gateway_dir: &Path) -> Result<()> {
|
||||
let profile =
|
||||
NifBuildProfile::from_env_value(env::var("FLUXER_GATEWAY_NIF_PROFILE").ok().as_deref());
|
||||
let builds = gateway_nif_builds(
|
||||
gateway_dir,
|
||||
profile,
|
||||
env::consts::DLL_PREFIX,
|
||||
env::consts::DLL_EXTENSION,
|
||||
);
|
||||
let priv_dir = gateway_dir.join("priv");
|
||||
fs::create_dir_all(&priv_dir)
|
||||
.with_context(|| format!("Failed to create {}", priv_dir.display()))?;
|
||||
|
||||
for build in builds {
|
||||
let mut cargo_args = build.cargo_args.clone();
|
||||
cargo_args.push(OsString::from("--manifest-path"));
|
||||
cargo_args.push(build.native_dir.join("Cargo.toml").into_os_string());
|
||||
run_command(CommandSpec::new("cargo").args(cargo_args))?;
|
||||
|
||||
ensure!(
|
||||
build.artifact_path.is_file(),
|
||||
"Expected NIF artifact was not produced: {}",
|
||||
build.artifact_path.display()
|
||||
);
|
||||
fs::copy(&build.artifact_path, &build.output_path).with_context(|| {
|
||||
format!(
|
||||
"Failed to copy {} to {}",
|
||||
build.artifact_path.display(),
|
||||
build.output_path.display()
|
||||
)
|
||||
})?;
|
||||
println!(
|
||||
"Installed gateway NIF {} -> {}",
|
||||
build.crate_name,
|
||||
build.output_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_gateway_check_step(label: &str, run: impl FnOnce() -> Result<()>) -> Result<()> {
|
||||
println!("========================================");
|
||||
println!(" {label}");
|
||||
println!("========================================");
|
||||
run()?;
|
||||
println!();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_rebar(
|
||||
gateway_dir: &Path,
|
||||
args: impl IntoIterator<Item = impl Into<OsString>>,
|
||||
) -> Result<()> {
|
||||
run_command(rebar_command(gateway_dir, args))
|
||||
}
|
||||
|
||||
fn run_gateway_fmt_check(gateway_dir: &Path) -> Result<()> {
|
||||
ensure!(
|
||||
command_succeeds(with_asdf_shims(CommandSpec::new("rebar3").arg("--version"))),
|
||||
"rebar3 is required for gateway formatting"
|
||||
);
|
||||
let output = crate::common::capture(rebar_command(gateway_dir, ["fmt", "--check"]))?;
|
||||
if output.status == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let combined = format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
if combined.contains("Command fmt not found")
|
||||
|| combined.to_ascii_lowercase().contains("not found")
|
||||
{
|
||||
println!("rebar3 fmt plugin is not configured; skipping gateway formatting check.");
|
||||
return Ok(());
|
||||
}
|
||||
bail!("gateway formatting failed with exit code {}", output.status)
|
||||
}
|
||||
|
||||
fn run_eqwalizer(gateway_dir: &Path, profile: &str) -> Result<()> {
|
||||
ensure!(
|
||||
!profile.is_empty(),
|
||||
"--eqwalizer-profile requires a non-empty profile name"
|
||||
);
|
||||
ensure!(
|
||||
command_succeeds(with_asdf_shims(CommandSpec::new("elp").arg("version"))),
|
||||
"elp not found in PATH. Install ELP from https://github.com/WhatsApp/erlang-language-platform/releases"
|
||||
);
|
||||
ensure!(
|
||||
command_succeeds(with_asdf_shims(CommandSpec::new("erl").arg("-version"))),
|
||||
"erl not found in PATH"
|
||||
);
|
||||
let erlang_source = find_erlang_source()?;
|
||||
let elp_version = output_text(with_asdf_shims(CommandSpec::new("elp").arg("version")))?;
|
||||
println!("==> Running eqWAlizer with {elp_version}");
|
||||
println!("==> Using Erlang source: {erlang_source}");
|
||||
println!("==> Rebar profile: {profile}");
|
||||
run_command(
|
||||
with_asdf_shims(
|
||||
CommandSpec::new("elp")
|
||||
.args([
|
||||
"eqwalize-all",
|
||||
"--rebar",
|
||||
"--as",
|
||||
profile,
|
||||
"--stats",
|
||||
"--bail-on-error",
|
||||
])
|
||||
.current_dir(gateway_dir),
|
||||
)
|
||||
.env("REBAR_SKIP_PROJECT_PLUGINS", "1"),
|
||||
)
|
||||
}
|
||||
|
||||
fn find_erlang_source() -> Result<String> {
|
||||
output_text(with_asdf_shims(CommandSpec::new("erl").args([
|
||||
"-noshell",
|
||||
"-eval",
|
||||
concat!(
|
||||
"Root = code:root_dir(), ",
|
||||
"Matches = filelib:wildcard(filename:join([Root, \"lib\", \"erts-*\", \"src\", \"erlang.erl\"])), ",
|
||||
"case Matches of ",
|
||||
"[Path | _] -> io:format(\"~s~n\", [Path]), halt(0); ",
|
||||
"[] -> halt(2) ",
|
||||
"end."
|
||||
),
|
||||
])))
|
||||
.context(
|
||||
"Erlang/OTP source files are required for Eqwalizer. On Debian/Ubuntu, install erlang-src",
|
||||
)
|
||||
}
|
||||
|
||||
fn rebar_command(
|
||||
gateway_dir: &Path,
|
||||
args: impl IntoIterator<Item = impl Into<OsString>>,
|
||||
) -> CommandSpec {
|
||||
let args = args.into_iter().map(Into::into).collect::<Vec<_>>();
|
||||
let should_skip_plugins = should_skip_rebar_project_plugins(&args);
|
||||
let mut spec = with_asdf_shims(
|
||||
CommandSpec::new("rebar3")
|
||||
.args(args)
|
||||
.current_dir(gateway_dir),
|
||||
);
|
||||
if should_skip_plugins {
|
||||
spec = spec.env("REBAR_SKIP_PROJECT_PLUGINS", "1");
|
||||
}
|
||||
spec
|
||||
}
|
||||
|
||||
fn should_skip_rebar_project_plugins(args: &[OsString]) -> bool {
|
||||
!args
|
||||
.iter()
|
||||
.any(|arg| matches!(arg.to_string_lossy().as_ref(), "fmt" | "lint" | "plugins"))
|
||||
}
|
||||
|
||||
fn with_asdf_shims(spec: CommandSpec) -> CommandSpec {
|
||||
let Some(shims_path) = asdf_shims_path() else {
|
||||
return spec;
|
||||
};
|
||||
if !shims_path.is_dir() {
|
||||
return spec;
|
||||
}
|
||||
let path = env::var_os("PATH").unwrap_or_default();
|
||||
let mut paths = std::iter::once(shims_path)
|
||||
.chain(env::split_paths(&path))
|
||||
.collect::<Vec<_>>();
|
||||
let joined = env::join_paths(paths.drain(..)).unwrap_or(path);
|
||||
spec.env("PATH", joined)
|
||||
}
|
||||
|
||||
fn asdf_shims_path() -> Option<PathBuf> {
|
||||
if let Some(asdf_data_dir) = env::var_os("ASDF_DATA_DIR") {
|
||||
return Some(PathBuf::from(asdf_data_dir).join("shims"));
|
||||
}
|
||||
env::var_os("HOME")
|
||||
.filter(|home| !home.is_empty())
|
||||
.map(|home| PathBuf::from(home).join(".asdf/shims"))
|
||||
}
|
||||
|
||||
fn gateway_nif_builds(
|
||||
gateway_dir: &Path,
|
||||
profile: NifBuildProfile,
|
||||
dll_prefix: &str,
|
||||
dll_extension: &str,
|
||||
) -> Vec<GatewayNifBuild> {
|
||||
GATEWAY_NIF_CRATES
|
||||
.iter()
|
||||
.map(|crate_name| {
|
||||
gateway_nif_build(gateway_dir, profile, dll_prefix, dll_extension, crate_name)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn gateway_nif_build(
|
||||
gateway_dir: &Path,
|
||||
profile: NifBuildProfile,
|
||||
dll_prefix: &str,
|
||||
dll_extension: &str,
|
||||
crate_name: &str,
|
||||
) -> GatewayNifBuild {
|
||||
let native_dir = gateway_dir.join("native").join(crate_name);
|
||||
GatewayNifBuild {
|
||||
crate_name: crate_name.to_string(),
|
||||
cargo_args: profile.cargo_args(),
|
||||
artifact_path: native_dir
|
||||
.join("target")
|
||||
.join(profile.target_dir_name())
|
||||
.join(format!("{dll_prefix}{crate_name}.{dll_extension}")),
|
||||
output_path: gateway_dir.join("priv").join(format!("{crate_name}.so")),
|
||||
native_dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_gateway_dir() -> Result<PathBuf> {
|
||||
let cwd = env::current_dir().context("Failed to resolve current directory")?;
|
||||
if cwd.join("rebar.config").is_file()
|
||||
&& cwd.file_name().and_then(|value| value.to_str()) == Some("fluxer_gateway")
|
||||
{
|
||||
return Ok(cwd);
|
||||
}
|
||||
if cwd.join("fluxer_gateway/rebar.config").is_file() {
|
||||
return Ok(cwd.join("fluxer_gateway"));
|
||||
}
|
||||
Err(anyhow!(
|
||||
"Could not resolve fluxer_gateway directory from {}",
|
||||
cwd.display()
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn nif_profile_defaults_to_release_and_uses_debug_for_other_values() {
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(None),
|
||||
NifBuildProfile::Release
|
||||
);
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(Some("release")),
|
||||
NifBuildProfile::Release
|
||||
);
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(Some("debug")),
|
||||
NifBuildProfile::Debug
|
||||
);
|
||||
assert_eq!(
|
||||
NifBuildProfile::from_env_value(Some("dev")),
|
||||
NifBuildProfile::Debug
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_nif_build_plan_matches_legacy_artifact_layout() {
|
||||
let gateway_dir = Path::new("/repo/fluxer_gateway");
|
||||
let builds = gateway_nif_builds(gateway_dir, NifBuildProfile::Release, "lib", "so");
|
||||
|
||||
assert_eq!(builds.len(), 2);
|
||||
assert_eq!(
|
||||
builds[0],
|
||||
GatewayNifBuild {
|
||||
crate_name: "push_markdown_plaintext_nif".to_string(),
|
||||
native_dir: PathBuf::from(
|
||||
"/repo/fluxer_gateway/native/push_markdown_plaintext_nif"
|
||||
),
|
||||
cargo_args: vec![OsString::from("build"), OsString::from("--release")],
|
||||
artifact_path: PathBuf::from(
|
||||
"/repo/fluxer_gateway/native/push_markdown_plaintext_nif/target/release/libpush_markdown_plaintext_nif.so"
|
||||
),
|
||||
output_path: PathBuf::from(
|
||||
"/repo/fluxer_gateway/priv/push_markdown_plaintext_nif.so"
|
||||
),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_build_plan_omits_release_arg_and_uses_debug_target_dir() {
|
||||
let build = gateway_nif_build(
|
||||
Path::new("/repo/fluxer_gateway"),
|
||||
NifBuildProfile::Debug,
|
||||
"lib",
|
||||
"dylib",
|
||||
"guild_member_list_oset_nif",
|
||||
);
|
||||
|
||||
assert_eq!(build.cargo_args, vec![OsString::from("build")]);
|
||||
assert_eq!(
|
||||
build.artifact_path,
|
||||
PathBuf::from(
|
||||
"/repo/fluxer_gateway/native/guild_member_list_oset_nif/target/debug/libguild_member_list_oset_nif.dylib"
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
build.output_path,
|
||||
PathBuf::from("/repo/fluxer_gateway/priv/guild_member_list_oset_nif.so")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebar_project_plugins_are_skipped_except_for_plugin_commands() {
|
||||
assert!(should_skip_rebar_project_plugins(&[OsString::from(
|
||||
"compile"
|
||||
)]));
|
||||
assert!(should_skip_rebar_project_plugins(&[
|
||||
OsString::from("as"),
|
||||
OsString::from("test"),
|
||||
OsString::from("eunit"),
|
||||
]));
|
||||
assert!(!should_skip_rebar_project_plugins(&[OsString::from("fmt")]));
|
||||
assert!(!should_skip_rebar_project_plugins(&[OsString::from(
|
||||
"lint"
|
||||
)]));
|
||||
assert!(!should_skip_rebar_project_plugins(&[OsString::from(
|
||||
"plugins"
|
||||
)]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebar_command_runs_in_gateway_dir_and_sets_skip_env_for_compile() {
|
||||
let command = rebar_command(Path::new("/repo/fluxer_gateway"), ["compile"]);
|
||||
|
||||
assert_eq!(command.program, OsString::from("rebar3"));
|
||||
assert_eq!(command.args, vec![OsString::from("compile")]);
|
||||
assert_eq!(command.cwd, Some(PathBuf::from("/repo/fluxer_gateway")));
|
||||
assert!(command.env.contains(&(
|
||||
OsString::from("REBAR_SKIP_PROJECT_PLUGINS"),
|
||||
OsString::from("1")
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebar_command_keeps_project_plugins_for_fmt() {
|
||||
let command = rebar_command(Path::new("/repo/fluxer_gateway"), ["fmt", "--check"]);
|
||||
|
||||
assert_eq!(
|
||||
command.args,
|
||||
vec![OsString::from("fmt"), OsString::from("--check")]
|
||||
);
|
||||
assert!(
|
||||
!command
|
||||
.env
|
||||
.iter()
|
||||
.any(|(key, _)| key == &OsString::from("REBAR_SKIP_PROJECT_PLUGINS"))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
mod app_dev_server;
|
||||
mod app_proxy;
|
||||
mod app_wasm;
|
||||
mod calver;
|
||||
mod ci_workflow;
|
||||
mod common;
|
||||
mod desktop;
|
||||
mod desktop_native;
|
||||
mod functions;
|
||||
mod gateway;
|
||||
mod release;
|
||||
mod schema;
|
||||
mod static_bucket;
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(name = "fluxer-ci")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum Command {
|
||||
AppDevServer(app_dev_server::AppDevServerArgs),
|
||||
BuildAppWasm(app_wasm::BuildAppWasmArgs),
|
||||
BuildAppProxy(app_proxy::BuildAppProxyArgs),
|
||||
BuildMarkdownParserWasm(app_wasm::BuildMarkdownParserWasmArgs),
|
||||
BuildDesktop(desktop::BuildDesktopArgs),
|
||||
BuildDesktopNativeAddon(desktop_native::BuildDesktopNativeAddonArgs),
|
||||
BuildGatewayNifs(gateway::BuildGatewayNifsArgs),
|
||||
Ci(ci_workflow::CiArgs),
|
||||
CiScripts(ci_workflow::CiScriptsArgs),
|
||||
CleanSchemaGeneratedFiles(schema::CleanSchemaGeneratedFilesArgs),
|
||||
Gateway(gateway::GatewayArgs),
|
||||
RepairStaticAssetMetadata(static_bucket::RepairStaticAssetMetadataArgs),
|
||||
Release(release::ReleaseArgs),
|
||||
ResolveCalver(calver::ResolveCalverArgs),
|
||||
SyncStaticBucket(static_bucket::SyncStaticBucketArgs),
|
||||
TestWebrtcSenderRust(desktop_native::TestWebrtcSenderRustArgs),
|
||||
}
|
||||
|
||||
pub async fn run() -> Result<()> {
|
||||
match Cli::parse().command {
|
||||
Command::AppDevServer(args) => app_dev_server::run(args).await,
|
||||
Command::BuildAppWasm(args) => app_wasm::run_build_app_wasm(args),
|
||||
Command::BuildAppProxy(args) => app_proxy::run(args).await,
|
||||
Command::BuildMarkdownParserWasm(args) => app_wasm::run_build_markdown_parser_wasm(args),
|
||||
Command::BuildDesktop(args) => desktop::run(args).await,
|
||||
Command::BuildDesktopNativeAddon(args) => {
|
||||
desktop_native::run_build_desktop_native_addon(args)
|
||||
}
|
||||
Command::BuildGatewayNifs(args) => gateway::run_build_gateway_nifs(args),
|
||||
Command::Ci(args) => ci_workflow::run_ci(args).await,
|
||||
Command::CiScripts(args) => ci_workflow::run_ci_scripts(args).await,
|
||||
Command::CleanSchemaGeneratedFiles(args) => schema::run_clean_generated_files(args),
|
||||
Command::Gateway(args) => gateway::run_gateway(args),
|
||||
Command::RepairStaticAssetMetadata(args) => {
|
||||
static_bucket::repair_asset_metadata(args).await
|
||||
}
|
||||
Command::Release(args) => release::run(args).await,
|
||||
Command::ResolveCalver(args) => calver::run(args),
|
||||
Command::SyncStaticBucket(args) => static_bucket::run(args).await,
|
||||
Command::TestWebrtcSenderRust(args) => desktop_native::run_test_webrtc_sender_rust(args),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
if let Err(error) = fluxer_ci::run().await {
|
||||
eprintln!("{error:?}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,115 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::collect_files;
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Args;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct CleanSchemaGeneratedFilesArgs {
|
||||
#[arg(long, default_value = "packages/schema/src/gen")]
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
pub fn run_clean_generated_files(args: CleanSchemaGeneratedFilesArgs) -> Result<()> {
|
||||
clean_generated_files(&args.root)
|
||||
}
|
||||
|
||||
fn clean_generated_files(root: &Path) -> Result<()> {
|
||||
for file in collect_files(root)?
|
||||
.into_iter()
|
||||
.filter(|path| path.extension().and_then(|value| value.to_str()) == Some("ts"))
|
||||
{
|
||||
let source = fs::read_to_string(&file)
|
||||
.with_context(|| format!("Failed to read {}", file.display()))?;
|
||||
let Some(import_index) = find_import_start(&source) else {
|
||||
continue;
|
||||
};
|
||||
let content = format!(
|
||||
"{}\n",
|
||||
collapse_extra_blank_lines(source[import_index..].trim_end())
|
||||
);
|
||||
if content != source {
|
||||
fs::write(&file, content)
|
||||
.with_context(|| format!("Failed to write {}", file.display()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn find_import_start(source: &str) -> Option<usize> {
|
||||
let mut offset = 0usize;
|
||||
for line in source.split_inclusive('\n') {
|
||||
if line.starts_with("import ") {
|
||||
return Some(offset);
|
||||
}
|
||||
offset += line.len();
|
||||
}
|
||||
if source[offset..].starts_with("import ") {
|
||||
return Some(offset);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn collapse_extra_blank_lines(source: &str) -> String {
|
||||
let mut output = String::with_capacity(source.len());
|
||||
let mut consecutive_newlines = 0usize;
|
||||
for ch in source.chars() {
|
||||
if ch == '\n' {
|
||||
consecutive_newlines += 1;
|
||||
if consecutive_newlines <= 2 {
|
||||
output.push(ch);
|
||||
}
|
||||
} else {
|
||||
consecutive_newlines = 0;
|
||||
output.push(ch);
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn write_file(path: &Path, contents: &str) {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_import_start_only_matches_line_starts() {
|
||||
assert_eq!(
|
||||
find_import_start("// import nope\nimport {x} from 'x';\n"),
|
||||
Some(15)
|
||||
);
|
||||
assert_eq!(find_import_start("const value = 'import nope';\n"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collapse_extra_blank_lines_keeps_at_most_one_blank_line() {
|
||||
assert_eq!(collapse_extra_blank_lines("a\n\n\n\nb\n\nc"), "a\n\nb\n\nc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_generated_files_removes_prelude_and_compacts_blank_lines() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path();
|
||||
let file = root.join("generated.ts");
|
||||
write_file(
|
||||
&file,
|
||||
"// header\n\n\nimport {x} from 'x';\n\n\n\nexport const y = x;\n\n",
|
||||
);
|
||||
write_file(&root.join("keep.txt"), "// header\n");
|
||||
|
||||
clean_generated_files(root).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
fs::read_to_string(file).unwrap(),
|
||||
"import {x} from 'x';\n\nexport const y = x;\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::common::{
|
||||
S3UploadPlanItem, collect_files, delete_s3_objects, list_s3_keys, path_to_s3_key,
|
||||
replace_s3_object_metadata, s3_client, s3_content_type_for_key, upload_s3_plan_sync,
|
||||
};
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use clap::Args;
|
||||
use std::collections::BTreeMap;
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
const DEFAULT_SOURCE: &str = "fluxer_static";
|
||||
const DEFAULT_STATIC_BUCKET: &str = "fluxer-static";
|
||||
const DEFAULT_S3_ENDPOINT: &str = "https://ewr1.vultrobjects.com";
|
||||
const DEFAULT_ASSET_PREFIX: &str = "assets/";
|
||||
const IMMUTABLE_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable";
|
||||
const DEFAULT_REPAIR_CONCURRENCY: usize = 8;
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct SyncStaticBucketArgs {
|
||||
#[arg(long, default_value = DEFAULT_SOURCE)]
|
||||
source: PathBuf,
|
||||
#[arg(long, default_value = DEFAULT_STATIC_BUCKET)]
|
||||
bucket: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone)]
|
||||
pub struct RepairStaticAssetMetadataArgs {
|
||||
#[arg(long, default_value = DEFAULT_STATIC_BUCKET)]
|
||||
bucket: String,
|
||||
#[arg(long, default_value = DEFAULT_ASSET_PREFIX)]
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
pub async fn run(args: SyncStaticBucketArgs) -> Result<()> {
|
||||
ensure!(
|
||||
args.source.is_dir(),
|
||||
"Static source directory is missing: {}",
|
||||
args.source.display()
|
||||
);
|
||||
|
||||
let upload_plan = static_upload_plan(&args.source)?;
|
||||
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
|
||||
|
||||
let plan = upload_plan
|
||||
.into_iter()
|
||||
.map(|(key, path)| S3UploadPlanItem::new(path, key).with_detected_content_type())
|
||||
.collect::<Vec<_>>();
|
||||
let stats = upload_s3_plan_sync(&client, &args.bucket, plan).await?;
|
||||
|
||||
println!(
|
||||
"Static bucket sync complete: uploaded {} file(s), skipped existing {}",
|
||||
stats.uploaded, stats.skipped_existing
|
||||
);
|
||||
|
||||
let remote_keys = list_s3_keys(&client, &args.bucket, "").await?;
|
||||
let markdown_keys = remote_keys
|
||||
.into_iter()
|
||||
.filter(|key| key.to_ascii_lowercase().ends_with(".md"))
|
||||
.collect::<Vec<_>>();
|
||||
let removed = delete_s3_objects(&client, &args.bucket, &markdown_keys).await?;
|
||||
println!("Static bucket sync removed {removed} stray .md object(s)");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn repair_asset_metadata(args: RepairStaticAssetMetadataArgs) -> Result<()> {
|
||||
let client = s3_client(Some(DEFAULT_S3_ENDPOINT)).await?;
|
||||
let keys = list_s3_keys(&client, &args.bucket, &args.prefix).await?;
|
||||
let mut skipped = 0_usize;
|
||||
let mut tasks = JoinSet::new();
|
||||
let semaphore = Arc::new(Semaphore::new(static_asset_repair_concurrency()));
|
||||
|
||||
for key in keys {
|
||||
let Some(content_type) = s3_content_type_for_key(&key) else {
|
||||
skipped += 1;
|
||||
continue;
|
||||
};
|
||||
let permit = semaphore
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.context("S3 repair semaphore closed")?;
|
||||
let client = client.clone();
|
||||
let bucket = args.bucket.clone();
|
||||
tasks.spawn(async move {
|
||||
let _permit = permit;
|
||||
replace_s3_object_metadata(
|
||||
&client,
|
||||
&bucket,
|
||||
&key,
|
||||
Some(content_type),
|
||||
Some(IMMUTABLE_ASSET_CACHE_CONTROL),
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
});
|
||||
}
|
||||
|
||||
let mut repaired = 0_usize;
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
result.context("S3 metadata repair task failed")??;
|
||||
repaired += 1;
|
||||
}
|
||||
|
||||
println!(
|
||||
"Static asset metadata repair complete: repaired {repaired} file(s), skipped {skipped}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn static_asset_repair_concurrency() -> usize {
|
||||
env::var("S3_WRITE_CONCURRENCY")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(DEFAULT_REPAIR_CONCURRENCY)
|
||||
}
|
||||
|
||||
fn static_upload_plan(source: &Path) -> Result<BTreeMap<String, PathBuf>> {
|
||||
let mut plan = BTreeMap::new();
|
||||
for file in collect_files(source)? {
|
||||
let relative = file
|
||||
.strip_prefix(source)
|
||||
.with_context(|| format!("Failed to relativize {}", file.display()))?;
|
||||
if !should_sync_static_path(relative) {
|
||||
continue;
|
||||
}
|
||||
let key = path_to_s3_key(relative);
|
||||
if !key.is_empty() {
|
||||
plan.insert(key, file);
|
||||
}
|
||||
}
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
fn should_sync_static_path(relative: &Path) -> bool {
|
||||
if relative
|
||||
.extension()
|
||||
.and_then(|ext| ext.to_str())
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("md"))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(first) = relative.components().next() else {
|
||||
return false;
|
||||
};
|
||||
match first {
|
||||
std::path::Component::Normal(value) => {
|
||||
let value = value.to_string_lossy();
|
||||
value != ".github" && value != "assets"
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
fn write_file(path: &Path, contents: &str) {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_plan_excludes_github_and_assets_roots() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let source = temp.path();
|
||||
write_file(&source.join("index.html"), "html");
|
||||
write_file(&source.join("docs/install.html"), "docs");
|
||||
write_file(&source.join(".github/workflows/ignored.yaml"), "workflow");
|
||||
write_file(&source.join("assets/app.js"), "asset");
|
||||
|
||||
let keys = static_upload_plan(source)
|
||||
.unwrap()
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(keys, vec!["docs/install.html", "index.html"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_plan_is_deterministic_and_keeps_non_root_assets_paths() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let source = temp.path();
|
||||
write_file(&source.join("z.html"), "z");
|
||||
write_file(&source.join("docs/assets/keep.js"), "keep");
|
||||
write_file(&source.join("a.html"), "a");
|
||||
|
||||
let keys = static_upload_plan(source)
|
||||
.unwrap()
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(keys, vec!["a.html", "docs/assets/keep.js", "z.html"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn static_key_filter_matches_workflow_excludes() {
|
||||
assert!(should_sync_static_path(Path::new("index.html")));
|
||||
assert!(should_sync_static_path(
|
||||
Path::new("docs").join("install.html").as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new("assets").join("app.js").as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new(".github")
|
||||
.join("workflows")
|
||||
.join("sync.yaml")
|
||||
.as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(Path::new("")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn static_key_filter_excludes_markdown() {
|
||||
assert!(!should_sync_static_path(Path::new(
|
||||
"THIRD_PARTY_LICENSES.md"
|
||||
)));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new("fonts").join("NOTICE.md").as_path()
|
||||
));
|
||||
assert!(!should_sync_static_path(
|
||||
Path::new("emoji").join("README.MD").as_path()
|
||||
));
|
||||
assert!(should_sync_static_path(Path::new("index.html")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_plan_excludes_markdown_files() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let source = temp.path();
|
||||
write_file(&source.join("index.html"), "html");
|
||||
write_file(&source.join("THIRD_PARTY_LICENSES.md"), "licenses");
|
||||
write_file(&source.join("fonts/NOTICE.md"), "notice");
|
||||
|
||||
let keys = static_upload_plan(source)
|
||||
.unwrap()
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(keys, vec!["index.html"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repair_metadata_args_default_to_app_assets_prefix() {
|
||||
let args = RepairStaticAssetMetadataArgs {
|
||||
bucket: DEFAULT_STATIC_BUCKET.to_string(),
|
||||
prefix: DEFAULT_ASSET_PREFIX.to_string(),
|
||||
};
|
||||
assert_eq!(args.bucket, "fluxer-static");
|
||||
assert_eq!(args.prefix, "assets/");
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
export interface RgbaTransformResult {
|
||||
rgba: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface DecodedFrame {
|
||||
rgba: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
delayMs: number;
|
||||
}
|
||||
|
||||
export interface EncodedApngFrame {
|
||||
compressed: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
delayMs: number;
|
||||
}
|
||||
|
||||
export interface EncodedGifChunk {
|
||||
data: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export function assemble_apng_frames(frames: Array<EncodedApngFrame>): Uint8Array;
|
||||
export function assemble_gif_frame_chunks(chunks: Array<EncodedGifChunk>): Uint8Array;
|
||||
export function crop_and_rotate_apng(
|
||||
input: Uint8Array,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): Uint8Array;
|
||||
export function crop_and_rotate_gif(
|
||||
input: Uint8Array,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): Uint8Array;
|
||||
export function crop_and_rotate_image(
|
||||
input: Uint8Array,
|
||||
format_hint: string,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): Uint8Array;
|
||||
export function crop_rotate_rgba(
|
||||
input: Uint8Array,
|
||||
src_width: number,
|
||||
src_height: number,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
rotation_deg: number,
|
||||
resize_width?: number | null,
|
||||
resize_height?: number | null,
|
||||
): RgbaTransformResult;
|
||||
export function decode_apng_frames(input: Uint8Array): Array<DecodedFrame>;
|
||||
export function decode_gif_frames(input: Uint8Array): Array<DecodedFrame>;
|
||||
export function encode_apng_frame_payload(frame: DecodedFrame): EncodedApngFrame;
|
||||
export function encode_apng_frames(frames: Array<DecodedFrame>): Uint8Array;
|
||||
export function encode_gif_frame_chunk(frame: DecodedFrame, first?: boolean): EncodedGifChunk;
|
||||
export function encode_gif_frames(frames: Array<DecodedFrame>): Uint8Array;
|
||||
@@ -0,0 +1,757 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {encode as encodePng} from 'fast-png';
|
||||
import {zlibSync} from 'fflate';
|
||||
import {applyPalette, GIFEncoder, quantize} from 'gifenc';
|
||||
import {decompressFrames, parseGIF} from 'gifuct-js';
|
||||
import {decode as decodeJpeg, encode as encodeJpeg} from 'jpeg-js';
|
||||
import UPNG from 'upng-js';
|
||||
|
||||
const textEncoder = new TextEncoder();
|
||||
const NULL_U32 = 0xffffffff;
|
||||
const RGBA_RESULT_HEADER_BYTES = 8;
|
||||
const MAX_ANIMATION_PIXELS = 200_000_000;
|
||||
const MAX_STATIC_DECODE_MEMORY_MB = 1024;
|
||||
|
||||
function inputBytes(input) {
|
||||
if (input == null) return new Uint8Array();
|
||||
return input instanceof Uint8Array ? input : new Uint8Array(input);
|
||||
}
|
||||
|
||||
function arrayBufferFor(bytes) {
|
||||
if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) return bytes.buffer;
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||
}
|
||||
|
||||
function nonNegativeU32(value) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number <= 0) return 0;
|
||||
return Math.min(NULL_U32, Math.floor(number));
|
||||
}
|
||||
|
||||
function cropCoordU32(value) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number <= 0) return 0;
|
||||
return Math.min(NULL_U32, Math.floor(number));
|
||||
}
|
||||
|
||||
function optionalDimension(value) {
|
||||
const dimension = nonNegativeU32(value);
|
||||
return dimension > 0 ? dimension : null;
|
||||
}
|
||||
|
||||
function effectiveRotation(rotationDeg) {
|
||||
const rotation = ((Math.floor(Number(rotationDeg) || 0) % 360) + 360) % 360;
|
||||
return rotation === 90 || rotation === 180 || rotation === 270 ? rotation : 0;
|
||||
}
|
||||
|
||||
function normalizedFormat(value) {
|
||||
const format = String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (format === 'jpg') return 'jpeg';
|
||||
if (format === 'apng') return 'png';
|
||||
if (format === 'animated_webp') return 'webp';
|
||||
return format;
|
||||
}
|
||||
|
||||
function sniffImageFormat(input) {
|
||||
const bytes = inputBytes(input);
|
||||
if (
|
||||
bytes.length >= 8 &&
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47 &&
|
||||
bytes[4] === 0x0d &&
|
||||
bytes[5] === 0x0a &&
|
||||
bytes[6] === 0x1a &&
|
||||
bytes[7] === 0x0a
|
||||
) {
|
||||
return 'png';
|
||||
}
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'jpeg';
|
||||
if (
|
||||
bytes.length >= 6 &&
|
||||
bytes[0] === 0x47 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x38 &&
|
||||
(bytes[4] === 0x37 || bytes[4] === 0x39) &&
|
||||
bytes[5] === 0x61
|
||||
) {
|
||||
return 'gif';
|
||||
}
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
bytes[0] === 0x52 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x46 &&
|
||||
bytes[8] === 0x57 &&
|
||||
bytes[9] === 0x45 &&
|
||||
bytes[10] === 0x42 &&
|
||||
bytes[11] === 0x50
|
||||
) {
|
||||
return 'webp';
|
||||
}
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
bytes[4] === 0x66 &&
|
||||
bytes[5] === 0x74 &&
|
||||
bytes[6] === 0x79 &&
|
||||
bytes[7] === 0x70 &&
|
||||
bytes[8] === 0x61 &&
|
||||
bytes[9] === 0x76 &&
|
||||
bytes[10] === 0x69 &&
|
||||
(bytes[11] === 0x66 || bytes[11] === 0x73)
|
||||
) {
|
||||
return 'avif';
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function readU32FromBytes(bytes, offset) {
|
||||
return new DataView(bytes.buffer, bytes.byteOffset + offset, 4).getUint32(0, true);
|
||||
}
|
||||
|
||||
function readU16LE(bytes, offset) {
|
||||
return bytes[offset] | (bytes[offset + 1] << 8);
|
||||
}
|
||||
|
||||
function readU16BE(bytes, offset) {
|
||||
return (bytes[offset] << 8) | bytes[offset + 1];
|
||||
}
|
||||
|
||||
function readU32BE(bytes, offset) {
|
||||
return (bytes[offset] * 0x1000000 + ((bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3])) >>> 0;
|
||||
}
|
||||
|
||||
function parseRgbaTransformResult(bytes) {
|
||||
if (bytes.byteLength < RGBA_RESULT_HEADER_BYTES) throw new Error('libfluxcore returned a truncated RGBA result');
|
||||
const width = readU32FromBytes(bytes, 0);
|
||||
const height = readU32FromBytes(bytes, 4);
|
||||
const expected = RGBA_RESULT_HEADER_BYTES + width * height * 4;
|
||||
if (bytes.byteLength !== expected) throw new Error('libfluxcore returned an invalid RGBA result length');
|
||||
return {rgba: bytes.subarray(RGBA_RESULT_HEADER_BYTES), width, height};
|
||||
}
|
||||
|
||||
function pngDimensions(bytes) {
|
||||
if (bytes.length < 24 || sniffImageFormat(bytes) !== 'png') return null;
|
||||
if (bytes[12] !== 0x49 || bytes[13] !== 0x48 || bytes[14] !== 0x44 || bytes[15] !== 0x52) return null;
|
||||
return {width: readU32BE(bytes, 16), height: readU32BE(bytes, 20)};
|
||||
}
|
||||
|
||||
function gifDimensions(bytes) {
|
||||
if (bytes.length < 10 || sniffImageFormat(bytes) !== 'gif') return null;
|
||||
return {width: readU16LE(bytes, 6), height: readU16LE(bytes, 8)};
|
||||
}
|
||||
|
||||
function jpegDimensions(bytes) {
|
||||
if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null;
|
||||
let offset = 2;
|
||||
while (offset + 4 <= bytes.length) {
|
||||
while (offset < bytes.length && bytes[offset] === 0xff) offset += 1;
|
||||
if (offset >= bytes.length) return null;
|
||||
const marker = bytes[offset];
|
||||
offset += 1;
|
||||
if (marker === 0xd9 || marker === 0xda) return null;
|
||||
if (offset + 2 > bytes.length) return null;
|
||||
const length = readU16BE(bytes, offset);
|
||||
if (length < 2 || offset + length > bytes.length) return null;
|
||||
if (
|
||||
marker === 0xc0 ||
|
||||
marker === 0xc1 ||
|
||||
marker === 0xc2 ||
|
||||
marker === 0xc3 ||
|
||||
marker === 0xc5 ||
|
||||
marker === 0xc6 ||
|
||||
marker === 0xc7 ||
|
||||
marker === 0xc9 ||
|
||||
marker === 0xca ||
|
||||
marker === 0xcb ||
|
||||
marker === 0xcd ||
|
||||
marker === 0xce ||
|
||||
marker === 0xcf
|
||||
) {
|
||||
if (length < 7) return null;
|
||||
return {height: readU16BE(bytes, offset + 3), width: readU16BE(bytes, offset + 5)};
|
||||
}
|
||||
offset += length;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function imageDimensions(bytes, format) {
|
||||
switch (format) {
|
||||
case 'png':
|
||||
return pngDimensions(bytes);
|
||||
case 'gif':
|
||||
return gifDimensions(bytes);
|
||||
case 'jpeg':
|
||||
return jpegDimensions(bytes);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isNoopTransform(imageWidth, imageHeight, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
const cropX = Math.min(cropCoordU32(x), imageWidth);
|
||||
const cropY = Math.min(cropCoordU32(y), imageHeight);
|
||||
const cropW = Math.min(nonNegativeU32(width), imageWidth - cropX);
|
||||
const cropH = Math.min(nonNegativeU32(height), imageHeight - cropY);
|
||||
const targetW = optionalDimension(resizeWidth) ?? imageWidth;
|
||||
const targetH = optionalDimension(resizeHeight) ?? imageHeight;
|
||||
return (
|
||||
cropX === 0 &&
|
||||
cropY === 0 &&
|
||||
cropW === imageWidth &&
|
||||
cropH === imageHeight &&
|
||||
effectiveRotation(rotationDeg) === 0 &&
|
||||
targetW === imageWidth &&
|
||||
targetH === imageHeight
|
||||
);
|
||||
}
|
||||
|
||||
export function crop_rotate_rgba(
|
||||
input,
|
||||
src_width,
|
||||
src_height,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotation_deg,
|
||||
resize_width,
|
||||
resize_height,
|
||||
) {
|
||||
const sourceWidth = nonNegativeU32(src_width);
|
||||
const sourceHeight = nonNegativeU32(src_height);
|
||||
return parseRgbaTransformResult(
|
||||
// biome-ignore lint/correctness/noUndeclaredVariables: Injected by the generated wasm wrapper at build time.
|
||||
crop_rotate_rgba_raw(
|
||||
inputBytes(input),
|
||||
sourceWidth,
|
||||
sourceHeight,
|
||||
cropCoordU32(x),
|
||||
cropCoordU32(y),
|
||||
nonNegativeU32(width),
|
||||
nonNegativeU32(height),
|
||||
effectiveRotation(rotation_deg),
|
||||
optionalDimension(resize_width),
|
||||
optionalDimension(resize_height),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function transformFrame(frame, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
const transformed = crop_rotate_rgba(
|
||||
frame.rgba,
|
||||
frame.width,
|
||||
frame.height,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotationDeg,
|
||||
resizeWidth,
|
||||
resizeHeight,
|
||||
);
|
||||
return {rgba: transformed.rgba, width: transformed.width, height: transformed.height, delayMs: frame.delayMs};
|
||||
}
|
||||
|
||||
function transformFrames(frames, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
let totalPixels = 0;
|
||||
const transformed = frames.map((frame) => {
|
||||
const out = transformFrame(frame, x, y, width, height, rotationDeg, resizeWidth, resizeHeight);
|
||||
totalPixels += out.width * out.height;
|
||||
if (totalPixels > MAX_ANIMATION_PIXELS) {
|
||||
throw new Error('Animated image is too large to crop. Try reducing its dimensions or number of frames.');
|
||||
}
|
||||
return out;
|
||||
});
|
||||
return transformed;
|
||||
}
|
||||
|
||||
function decodePngFrames(input) {
|
||||
const bytes = inputBytes(input);
|
||||
const decoded = UPNG.decode(arrayBufferFor(bytes));
|
||||
const rgbaFrames = UPNG.toRGBA8(decoded);
|
||||
if (!rgbaFrames.length) throw new Error('PNG has no frames');
|
||||
return rgbaFrames.map((frame, index) => {
|
||||
const delayMs = decoded.frames?.[index]?.delay ?? 0;
|
||||
return {rgba: new Uint8Array(frame), width: decoded.width, height: decoded.height, delayMs};
|
||||
});
|
||||
}
|
||||
|
||||
function decodeJpegFrame(input) {
|
||||
const decoded = decodeJpeg(inputBytes(input), {
|
||||
useTArray: true,
|
||||
formatAsRGBA: true,
|
||||
maxMemoryUsageInMB: MAX_STATIC_DECODE_MEMORY_MB,
|
||||
});
|
||||
return {rgba: new Uint8Array(decoded.data), width: decoded.width, height: decoded.height, delayMs: 0};
|
||||
}
|
||||
|
||||
function drawGifPatch(canvas, canvasWidth, frame) {
|
||||
const dims = frame.dims;
|
||||
const patch = frame.patch;
|
||||
for (let row = 0; row < dims.height; row += 1) {
|
||||
const canvasY = dims.top + row;
|
||||
if (canvasY < 0) continue;
|
||||
const canvasOffset = (canvasY * canvasWidth + dims.left) * 4;
|
||||
const patchOffset = row * dims.width * 4;
|
||||
if (canvasOffset < 0 || canvasOffset >= canvas.length) continue;
|
||||
for (let col = 0; col < dims.width; col += 1) {
|
||||
const source = patchOffset + col * 4;
|
||||
const target = canvasOffset + col * 4;
|
||||
if (target < 0 || target + 4 > canvas.length || source + 4 > patch.length) continue;
|
||||
if (patch[source + 3] === 0) continue;
|
||||
canvas[target] = patch[source];
|
||||
canvas[target + 1] = patch[source + 1];
|
||||
canvas[target + 2] = patch[source + 2];
|
||||
canvas[target + 3] = patch[source + 3];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function clearRectRgba(canvas, canvasWidth, x, y, width, height) {
|
||||
for (let row = 0; row < height; row += 1) {
|
||||
const start = ((y + row) * canvasWidth + x) * 4;
|
||||
const end = start + width * 4;
|
||||
if (start >= 0 && end <= canvas.length) canvas.fill(0, start, end);
|
||||
}
|
||||
}
|
||||
|
||||
function collectGifFrames(input, transformOptions) {
|
||||
const bytes = inputBytes(input);
|
||||
const parsed = parseGIF(arrayBufferFor(bytes));
|
||||
const screenWidth = parsed.lsd.width;
|
||||
const screenHeight = parsed.lsd.height;
|
||||
const decodedFrames = decompressFrames(parsed, true);
|
||||
if (!decodedFrames.length) throw new Error('GIF has no frames');
|
||||
const canvas = new Uint8Array(screenWidth * screenHeight * 4);
|
||||
let previousCanvas = null;
|
||||
const frames = [];
|
||||
for (const frame of decodedFrames) {
|
||||
if (frame.disposalType === 3) previousCanvas = canvas.slice();
|
||||
drawGifPatch(canvas, screenWidth, frame);
|
||||
const sourceFrame = {
|
||||
rgba: transformOptions ? canvas : canvas.slice(),
|
||||
width: screenWidth,
|
||||
height: screenHeight,
|
||||
delayMs: frame.delay || 100,
|
||||
};
|
||||
frames.push(
|
||||
transformOptions
|
||||
? transformFrame(
|
||||
sourceFrame,
|
||||
transformOptions.x,
|
||||
transformOptions.y,
|
||||
transformOptions.width,
|
||||
transformOptions.height,
|
||||
transformOptions.rotationDeg,
|
||||
transformOptions.resizeWidth,
|
||||
transformOptions.resizeHeight,
|
||||
)
|
||||
: sourceFrame,
|
||||
);
|
||||
if (frame.disposalType === 2) {
|
||||
clearRectRgba(canvas, screenWidth, frame.dims.left, frame.dims.top, frame.dims.width, frame.dims.height);
|
||||
} else if (frame.disposalType === 3 && previousCanvas) {
|
||||
canvas.set(previousCanvas);
|
||||
previousCanvas = null;
|
||||
}
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
function decodeGifFrames(input) {
|
||||
return collectGifFrames(input, null);
|
||||
}
|
||||
|
||||
function transformGifFrames(input, x, y, width, height, rotationDeg, resizeWidth, resizeHeight) {
|
||||
return collectGifFrames(input, {x, y, width, height, rotationDeg, resizeWidth, resizeHeight});
|
||||
}
|
||||
|
||||
function exactGifFrameData(rgba) {
|
||||
const palette = [];
|
||||
const colorToIndex = new Map();
|
||||
const index = new Uint8Array(rgba.length / 4);
|
||||
let transparentIndex = -1;
|
||||
for (let offset = 0, pixel = 0; offset < rgba.length; offset += 4, pixel += 1) {
|
||||
if (rgba[offset + 3] === 0) {
|
||||
if (transparentIndex === -1) {
|
||||
if (palette.length >= 256) return null;
|
||||
transparentIndex = palette.length;
|
||||
palette.push([0, 0, 0]);
|
||||
}
|
||||
index[pixel] = transparentIndex;
|
||||
continue;
|
||||
}
|
||||
const key = `${rgba[offset]},${rgba[offset + 1]},${rgba[offset + 2]}`;
|
||||
let paletteIndex = colorToIndex.get(key);
|
||||
if (paletteIndex == null) {
|
||||
if (palette.length >= 256) return null;
|
||||
paletteIndex = palette.length;
|
||||
colorToIndex.set(key, paletteIndex);
|
||||
palette.push([rgba[offset], rgba[offset + 1], rgba[offset + 2]]);
|
||||
}
|
||||
index[pixel] = paletteIndex;
|
||||
}
|
||||
if (palette.length === 0) palette.push([0, 0, 0]);
|
||||
return {index, palette, transparentIndex};
|
||||
}
|
||||
|
||||
function quantizedGifFrameData(rgba) {
|
||||
const input = rgba.byteOffset === 0 && rgba.byteLength === rgba.buffer.byteLength ? rgba : new Uint8Array(rgba);
|
||||
const palette = quantize(input, 256, {format: 'rgba4444', oneBitAlpha: true});
|
||||
const index = applyPalette(input, palette, 'rgba4444');
|
||||
const transparentIndex = palette.findIndex((color) => color.length >= 4 && color[3] === 0);
|
||||
const rgbPalette = palette.map((color) => [color[0], color[1], color[2]]);
|
||||
return {index, palette: rgbPalette, transparentIndex};
|
||||
}
|
||||
|
||||
function writeGifFrame(gif, frame, width, height, first) {
|
||||
if (frame.width !== width || frame.height !== height) throw new Error('GIF frame dimensions must match');
|
||||
const frameData = exactGifFrameData(frame.rgba) ?? quantizedGifFrameData(frame.rgba);
|
||||
const transparent = frameData.transparentIndex >= 0;
|
||||
gif.writeFrame(frameData.index, width, height, {
|
||||
palette: frameData.palette,
|
||||
delay: Math.max(0, Math.round(frame.delayMs || 0)),
|
||||
repeat: 0,
|
||||
transparent,
|
||||
transparentIndex: transparent ? frameData.transparentIndex : 0,
|
||||
first,
|
||||
});
|
||||
}
|
||||
|
||||
function encodeGifFrames(frames) {
|
||||
if (!frames.length) throw new Error('GIF encode requires at least one frame');
|
||||
const width = frames[0].width;
|
||||
const height = frames[0].height;
|
||||
const gif = GIFEncoder();
|
||||
for (const frame of frames) writeGifFrame(gif, frame, width, height, false);
|
||||
gif.finish();
|
||||
return gif.bytes();
|
||||
}
|
||||
|
||||
function encodeGifFrameChunk(frame, first) {
|
||||
const gif = GIFEncoder({auto: false});
|
||||
if (first) gif.writeHeader();
|
||||
writeGifFrame(gif, frame, frame.width, frame.height, first);
|
||||
return {data: gif.bytes(), width: frame.width, height: frame.height};
|
||||
}
|
||||
|
||||
function assembleGifFrameChunks(chunks) {
|
||||
if (!chunks.length) throw new Error('GIF chunk assembly requires at least one frame');
|
||||
const width = chunks[0].width;
|
||||
const height = chunks[0].height;
|
||||
const parts = [];
|
||||
for (const chunk of chunks) {
|
||||
if (chunk.width !== width || chunk.height !== height) throw new Error('GIF frame dimensions must match');
|
||||
parts.push(inputBytes(chunk.data));
|
||||
}
|
||||
parts.push(new Uint8Array([0x3b]));
|
||||
return concatBytes(parts);
|
||||
}
|
||||
|
||||
function writeU32BE(bytes, offset, value) {
|
||||
bytes[offset] = (value >>> 24) & 0xff;
|
||||
bytes[offset + 1] = (value >>> 16) & 0xff;
|
||||
bytes[offset + 2] = (value >>> 8) & 0xff;
|
||||
bytes[offset + 3] = value & 0xff;
|
||||
}
|
||||
|
||||
function writeU16BE(bytes, offset, value) {
|
||||
bytes[offset] = (value >>> 8) & 0xff;
|
||||
bytes[offset + 1] = value & 0xff;
|
||||
}
|
||||
|
||||
let pngCrcTable = null;
|
||||
function crc32(bytes, start, end) {
|
||||
if (!pngCrcTable) {
|
||||
pngCrcTable = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n += 1) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
pngCrcTable[n] = c >>> 0;
|
||||
}
|
||||
}
|
||||
let c = 0xffffffff;
|
||||
for (let index = start; index < end; index += 1) c = pngCrcTable[(c ^ bytes[index]) & 0xff] ^ (c >>> 8);
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function pngChunk(type, payload) {
|
||||
const typeBytes = textEncoder.encode(type);
|
||||
const chunk = new Uint8Array(12 + payload.length);
|
||||
writeU32BE(chunk, 0, payload.length);
|
||||
chunk.set(typeBytes, 4);
|
||||
chunk.set(payload, 8);
|
||||
writeU32BE(chunk, 8 + payload.length, crc32(chunk, 4, 8 + payload.length));
|
||||
return chunk;
|
||||
}
|
||||
|
||||
function concatBytes(parts) {
|
||||
let total = 0;
|
||||
for (const part of parts) total += part.length;
|
||||
const output = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
output.set(part, offset);
|
||||
offset += part.length;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function pngScanlines(rgba, width, height) {
|
||||
const rowBytes = width * 4;
|
||||
const output = new Uint8Array((rowBytes + 1) * height);
|
||||
for (let row = 0; row < height; row += 1) {
|
||||
const target = row * (rowBytes + 1);
|
||||
output[target] = 0;
|
||||
output.set(rgba.subarray(row * rowBytes, row * rowBytes + rowBytes), target + 1);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function delayFraction(delayMs) {
|
||||
const ms = Math.max(0, Math.round(Number(delayMs) || 0));
|
||||
if (ms === 0) return [0, 100];
|
||||
if (ms <= 655350) return [Math.min(65535, Math.max(1, Math.round(ms / 10))), 100];
|
||||
return [Math.min(65535, Math.max(1, Math.round(ms / 1000))), 1];
|
||||
}
|
||||
|
||||
function encodeApngFramePayload(frame) {
|
||||
return {
|
||||
compressed: zlibSync(pngScanlines(frame.rgba, frame.width, frame.height), {level: 6}),
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
delayMs: frame.delayMs,
|
||||
};
|
||||
}
|
||||
|
||||
function assembleApngFramePayloads(frames) {
|
||||
if (!frames.length) throw new Error('APNG encode requires at least one frame');
|
||||
const width = frames[0].width;
|
||||
const height = frames[0].height;
|
||||
const chunks = [new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])];
|
||||
const ihdr = new Uint8Array(13);
|
||||
writeU32BE(ihdr, 0, width);
|
||||
writeU32BE(ihdr, 4, height);
|
||||
ihdr[8] = 8;
|
||||
ihdr[9] = 6;
|
||||
chunks.push(pngChunk('IHDR', ihdr));
|
||||
const actl = new Uint8Array(8);
|
||||
writeU32BE(actl, 0, frames.length);
|
||||
writeU32BE(actl, 4, 0);
|
||||
chunks.push(pngChunk('acTL', actl));
|
||||
let sequence = 0;
|
||||
for (let index = 0; index < frames.length; index += 1) {
|
||||
const frame = frames[index];
|
||||
if (frame.width !== width || frame.height !== height) throw new Error('APNG frame dimensions must match');
|
||||
const fctl = new Uint8Array(26);
|
||||
writeU32BE(fctl, 0, sequence);
|
||||
sequence += 1;
|
||||
writeU32BE(fctl, 4, width);
|
||||
writeU32BE(fctl, 8, height);
|
||||
writeU32BE(fctl, 12, 0);
|
||||
writeU32BE(fctl, 16, 0);
|
||||
const delay = delayFraction(frame.delayMs);
|
||||
writeU16BE(fctl, 20, delay[0]);
|
||||
writeU16BE(fctl, 22, delay[1]);
|
||||
fctl[24] = 0;
|
||||
fctl[25] = 0;
|
||||
chunks.push(pngChunk('fcTL', fctl));
|
||||
if (index === 0) {
|
||||
chunks.push(pngChunk('IDAT', inputBytes(frame.compressed)));
|
||||
} else {
|
||||
const compressed = inputBytes(frame.compressed);
|
||||
const payload = new Uint8Array(4 + compressed.length);
|
||||
writeU32BE(payload, 0, sequence);
|
||||
sequence += 1;
|
||||
payload.set(compressed, 4);
|
||||
chunks.push(pngChunk('fdAT', payload));
|
||||
}
|
||||
}
|
||||
chunks.push(pngChunk('IEND', new Uint8Array()));
|
||||
return concatBytes(chunks);
|
||||
}
|
||||
|
||||
function encodeApngFrames(frames) {
|
||||
if (!frames.length) throw new Error('APNG encode requires at least one frame');
|
||||
const width = frames[0].width;
|
||||
const height = frames[0].height;
|
||||
for (const frame of frames) {
|
||||
if (frame.width !== width || frame.height !== height) throw new Error('APNG frame dimensions must match');
|
||||
}
|
||||
return assembleApngFramePayloads(frames.map(encodeApngFramePayload));
|
||||
}
|
||||
|
||||
function encodeStaticFrame(frame, outputFormat) {
|
||||
switch (outputFormat) {
|
||||
case 'png':
|
||||
return encodePng({width: frame.width, height: frame.height, data: frame.rgba, depth: 8, channels: 4});
|
||||
case 'jpeg':
|
||||
return encodeJpeg({width: frame.width, height: frame.height, data: frame.rgba}, 92).data;
|
||||
case 'gif':
|
||||
return encodeGifFrames([frame]);
|
||||
default:
|
||||
throw new Error(`Unsupported static output format: ${outputFormat}`);
|
||||
}
|
||||
}
|
||||
|
||||
function decodeStaticImage(input, inputFormat) {
|
||||
switch (inputFormat) {
|
||||
case 'png':
|
||||
return decodePngFrames(input)[0];
|
||||
case 'jpeg':
|
||||
return decodeJpegFrame(input);
|
||||
case 'gif':
|
||||
return decodeGifFrames(input)[0];
|
||||
default:
|
||||
throw new Error(`Unsupported static input format: ${inputFormat}`);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeFrameInput(frame) {
|
||||
if (!frame || frame.rgba == null) throw new Error('Frame is missing RGBA data');
|
||||
return {
|
||||
rgba: inputBytes(frame.rgba),
|
||||
width: nonNegativeU32(frame.width),
|
||||
height: nonNegativeU32(frame.height),
|
||||
delayMs: Math.max(0, Math.round(Number(frame.delayMs) || 0)),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeApngPayloadInput(frame) {
|
||||
if (!frame || frame.compressed == null) throw new Error('APNG frame is missing compressed data');
|
||||
return {
|
||||
compressed: inputBytes(frame.compressed),
|
||||
width: nonNegativeU32(frame.width),
|
||||
height: nonNegativeU32(frame.height),
|
||||
delayMs: Math.max(0, Math.round(Number(frame.delayMs) || 0)),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeGifChunkInput(chunk) {
|
||||
if (!chunk || chunk.data == null) throw new Error('GIF frame chunk is missing data');
|
||||
return {
|
||||
data: inputBytes(chunk.data),
|
||||
width: nonNegativeU32(chunk.width),
|
||||
height: nonNegativeU32(chunk.height),
|
||||
};
|
||||
}
|
||||
|
||||
export function decode_gif_frames(input) {
|
||||
return decodeGifFrames(input);
|
||||
}
|
||||
|
||||
export function decode_apng_frames(input) {
|
||||
return decodePngFrames(input);
|
||||
}
|
||||
|
||||
export function encode_gif_frames(frames) {
|
||||
return encodeGifFrames(frames.map(normalizeFrameInput));
|
||||
}
|
||||
|
||||
export function encode_apng_frames(frames) {
|
||||
return encodeApngFrames(frames.map(normalizeFrameInput));
|
||||
}
|
||||
|
||||
export function encode_gif_frame_chunk(frame, first) {
|
||||
return encodeGifFrameChunk(normalizeFrameInput(frame), Boolean(first));
|
||||
}
|
||||
|
||||
export function assemble_gif_frame_chunks(chunks) {
|
||||
return assembleGifFrameChunks(chunks.map(normalizeGifChunkInput));
|
||||
}
|
||||
|
||||
export function encode_apng_frame_payload(frame) {
|
||||
return encodeApngFramePayload(normalizeFrameInput(frame));
|
||||
}
|
||||
|
||||
export function assemble_apng_frames(frames) {
|
||||
return assembleApngFramePayloads(frames.map(normalizeApngPayloadInput));
|
||||
}
|
||||
|
||||
export function crop_and_rotate_apng(input, x, y, width, height, rotation_deg, resize_width, resize_height) {
|
||||
const bytes = inputBytes(input);
|
||||
const dimensions = pngDimensions(bytes);
|
||||
if (
|
||||
dimensions &&
|
||||
isNoopTransform(dimensions.width, dimensions.height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
const frames = decodePngFrames(bytes);
|
||||
if (
|
||||
isNoopTransform(frames[0].width, frames[0].height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
return encodeApngFrames(transformFrames(frames, x, y, width, height, rotation_deg, resize_width, resize_height));
|
||||
}
|
||||
|
||||
export function crop_and_rotate_gif(input, x, y, width, height, rotation_deg, resize_width, resize_height) {
|
||||
const bytes = inputBytes(input);
|
||||
const dimensions = gifDimensions(bytes);
|
||||
if (
|
||||
dimensions &&
|
||||
isNoopTransform(dimensions.width, dimensions.height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
return encodeGifFrames(transformGifFrames(bytes, x, y, width, height, rotation_deg, resize_width, resize_height));
|
||||
}
|
||||
|
||||
export function crop_and_rotate_image(
|
||||
input,
|
||||
format_hint,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotation_deg,
|
||||
resize_width,
|
||||
resize_height,
|
||||
) {
|
||||
const bytes = inputBytes(input);
|
||||
const inputFormat = sniffImageFormat(bytes);
|
||||
const requestedFormat = normalizedFormat(format_hint);
|
||||
const outputFormat = requestedFormat && requestedFormat !== 'unknown' ? requestedFormat : inputFormat;
|
||||
if (inputFormat === 'webp' || inputFormat === 'avif' || outputFormat === 'webp' || outputFormat === 'avif') {
|
||||
throw new Error('WebP and AVIF crop/encode use the browser or native media bridge');
|
||||
}
|
||||
if (inputFormat === outputFormat) {
|
||||
const dimensions = imageDimensions(bytes, inputFormat);
|
||||
if (
|
||||
dimensions &&
|
||||
isNoopTransform(
|
||||
dimensions.width,
|
||||
dimensions.height,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
rotation_deg,
|
||||
resize_width,
|
||||
resize_height,
|
||||
)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
}
|
||||
const frame = decodeStaticImage(bytes, inputFormat);
|
||||
if (
|
||||
inputFormat === outputFormat &&
|
||||
isNoopTransform(frame.width, frame.height, x, y, width, height, rotation_deg, resize_width, resize_height)
|
||||
) {
|
||||
return bytes.slice();
|
||||
}
|
||||
return encodeStaticFrame(
|
||||
transformFrame(frame, x, y, width, height, rotation_deg, resize_width, resize_height),
|
||||
outputFormat,
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user