Add native self-hosted instance connection to fluxer_desktop

Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
2026-07-01 18:22:43 -04:00
commit 682afacd30
1763 changed files with 613720 additions and 0 deletions
@@ -0,0 +1,108 @@
+// Microbenchmark: AVX-512 (zmm, 16-wide) element-wise activation kernels vs
+// their x86 predecessor.
+//
+//   sigmoid, tanh              : predecessor = FMA (256-bit, 8-wide) kernel
+//   hardswish, leaky_relu,
+//   silu, gelu                 : predecessor = generic scalar kernel
+//                                (no FMA kernel exists on x86)
+//
+// All buffers are 64-byte aligned (AVX-512 alignment_bytes) and a multiple of
+// 64 elements so every kernel's nr() divides the length. Criterion reports the
+// distribution; the min of the samples is the relevant "min-of-N" number.
+
+use criterion::*;
+use tract_data::prelude::*;
+use tract_linalg::element_wise::ElementWiseKer;
+
+const N: usize = 1024;
+
+fn aligned_input() -> Tensor {
+    let mut t = unsafe { Tensor::uninitialized_aligned::<f32>(&[N], 64).unwrap() };
+    let s = unsafe { t.as_slice_mut_unchecked::<f32>() };
+    for (i, x) in s.iter_mut().enumerate() {
+        *x = (i as f32 / 10.0).sin() * 5.0;
+    }
+    t
+}
+
+// Enable FTZ/DAZ (flush-to-zero, denormals-are-zero) for the whole process so
+// repeated in-place application of a kernel to its own output cannot collapse
+// into denormal arithmetic (extremely slow on x86) and distort the timing.
+// Mirrors what the sigmoid/tanh kernels already do internally via MXCSR.
+#[cfg(target_arch = "x86_64")]
+fn enable_ftz_daz() {
+    // Set MXCSR bit 15 (FTZ) and bit 6 (DAZ) directly; the safe intrinsic
+    // wrappers are deprecated in favour of inline asm.
+    unsafe {
+        let mut mxcsr: u32 = 0;
+        std::arch::asm!("stmxcsr [{p}]", p = in(reg) &mut mxcsr);
+        mxcsr |= (1 << 15) | (1 << 6);
+        std::arch::asm!("ldmxcsr [{p}]", p = in(reg) &mxcsr);
+    }
+}
+
+// In-place throughput, matching the convention of the existing element-wise
+// benches (sigmoid.rs / silu.rs).
+macro_rules! bench_pair {
+    ($c:expr, $name:expr, $pred_label:expr, $pred:ty, $avx512:ty $(, $param:expr)?) => {{
+        let mut group = $c.benchmark_group($name);
+        group.throughput(Throughput::Elements(N as u64));
+        let mut tp = aligned_input();
+        let sp = unsafe { tp.as_slice_mut_unchecked::<f32>() };
+        group.bench_function($pred_label, |b| {
+            b.iter(|| <$pred>::run(sp, ($($param)?)))
+        });
+        if std::is_x86_feature_detected!("avx512f") {
+            let mut ta = aligned_input();
+            let sa = unsafe { ta.as_slice_mut_unchecked::<f32>() };
+            group.bench_function("avx512", |b| {
+                b.iter(|| <$avx512>::run(sa, ($($param)?)))
+            });
+        }
+        group.finish();
+    }};
+}
+
+fn benches(c: &mut Criterion) {
+    #[cfg(target_arch = "x86_64")]
+    enable_ftz_daz();
+    use tract_linalg::x86_64_fma::act::*;
+    use tract_linalg::x86_64_fma::{
+        avx512_sigmoid_f32, avx512_tanh_f32, fma_sigmoid_f32, fma_tanh_f32,
+    };
+
+    bench_pair!(c, "sigmoid_f32", "fma", fma_sigmoid_f32, avx512_sigmoid_f32);
+    bench_pair!(c, "tanh_f32", "fma", fma_tanh_f32, avx512_tanh_f32);
+    bench_pair!(
+        c,
+        "hardswish_f32",
+        "generic",
+        tract_linalg::generic::SHardSwish4,
+        x86_64_avx512_hardswish_f32_64n
+    );
+    bench_pair!(
+        c,
+        "leaky_relu_f32",
+        "generic",
+        tract_linalg::generic::SLeakyRelu4,
+        x86_64_avx512_leaky_relu_f32_64n,
+        0.1f32
+    );
+    bench_pair!(
+        c,
+        "silu_f32",
+        "generic",
+        tract_linalg::generic::SSiLU4,
+        x86_64_avx512_silu_f32_16n
+    );
+    bench_pair!(
+        c,
+        "gelu_f32",
+        "generic",
+        tract_linalg::generic::SGelu4,
+        x86_64_avx512_gelu_f32_16n
+    );
+}
+
+criterion_group!(g, benches);
+criterion_main!(g);
@@ -0,0 +1,83 @@
+// Microbenchmark: AVX-512 f16 element-wise activations vs the generic scalar
+// f16 kernels (no FMA f16 predecessor exists on x86 — the generic baseline
+// already runs `(*v - max).to_f32()`-style per-element conversions). Buffers
+// are 64-byte aligned (alignment that the AVX-512 path uses internally) and
+// a multiple of 64 elements.
+
+use criterion::*;
+use tract_data::prelude::*;
+use tract_linalg::element_wise::ElementWiseKer;
+
+const N: usize = 1024;
+
+fn aligned_input() -> Tensor {
+    let mut t = unsafe { Tensor::uninitialized_aligned::<f16>(&[N], 64).unwrap() };
+    let s = unsafe { t.as_slice_mut_unchecked::<f16>() };
+    for (i, x) in s.iter_mut().enumerate() {
+        *x = f16::from_f32((i as f32 / 10.0).sin() * 5.0);
+    }
+    t
+}
+
+macro_rules! bench_pair {
+    ($c:expr, $name:expr, $pred:ty, $avx512:ty $(, $param:expr)?) => {{
+        let mut group = $c.benchmark_group($name);
+        group.throughput(Throughput::Elements(N as u64));
+        let mut tp = aligned_input();
+        let sp = unsafe { tp.as_slice_mut_unchecked::<f16>() };
+        group.bench_function("generic", |b| {
+            b.iter(|| <$pred>::run(sp, ($($param)?)))
+        });
+        if std::is_x86_feature_detected!("avx512f") {
+            let mut ta = aligned_input();
+            let sa = unsafe { ta.as_slice_mut_unchecked::<f16>() };
+            group.bench_function("avx512", |b| {
+                b.iter(|| <$avx512>::run(sa, ($($param)?)))
+            });
+        }
+        group.finish();
+    }};
+}
+
+fn benches(c: &mut Criterion) {
+    bench_pair!(
+        c,
+        "sigmoid_f16",
+        tract_linalg::generic::sigmoid::HSigmoid8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_sigmoid_f16_16n
+    );
+    bench_pair!(
+        c,
+        "tanh_f16",
+        tract_linalg::generic::tanh::HTanh8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_tanh_f16_16n
+    );
+    bench_pair!(
+        c,
+        "hardswish_f16",
+        tract_linalg::generic::hardswish::HHardSwish8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_hardswish_f16_64n
+    );
+    bench_pair!(
+        c,
+        "leaky_relu_f16",
+        tract_linalg::generic::leaky_relu::HLeakyRelu8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_leaky_relu_f16_64n,
+        f16::from_f32(0.1)
+    );
+    bench_pair!(
+        c,
+        "silu_f16",
+        tract_linalg::generic::silu::HSiLU8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_silu_f16_16n
+    );
+    bench_pair!(
+        c,
+        "gelu_f16",
+        tract_linalg::generic::gelu::HGelu8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_gelu_f16_16n
+    );
+}
+
+criterion_group!(g, benches);
+criterion_main!(g);
@@ -0,0 +1,68 @@
+// Microbench: AVX-512_FP16 native f16 element-wise activations vs the
+// f32-roundtrip versions in `act_f16.rs` (which were the AVX-512 f16 path
+// before native f16 ISA was available). Both run on 64-byte-aligned, 1024-
+// element buffers — same workload as the existing activations_avx512_f16
+// bench, just adding the native-fp16 column.
+
+use criterion::*;
+use tract_data::prelude::*;
+use tract_linalg::element_wise::ElementWiseKer;
+
+const N: usize = 1024;
+
+fn aligned_input() -> Tensor {
+    let mut t = unsafe { Tensor::uninitialized_aligned::<f16>(&[N], 64).unwrap() };
+    let s = unsafe { t.as_slice_mut_unchecked::<f16>() };
+    for (i, x) in s.iter_mut().enumerate() {
+        *x = f16::from_f32((i as f32 / 10.0).sin() * 5.0);
+    }
+    t
+}
+
+macro_rules! bench_triple {
+    ($c:expr, $name:expr, $pred:ty, $roundtrip:ty, $native:ty $(, $param:expr)?) => {{
+        let mut group = $c.benchmark_group($name);
+        group.throughput(Throughput::Elements(N as u64));
+        let mut tg = aligned_input();
+        let sg = unsafe { tg.as_slice_mut_unchecked::<f16>() };
+        group.bench_function("generic", |b| {
+            b.iter(|| <$pred>::run(sg, ($($param)?)))
+        });
+        if std::is_x86_feature_detected!("avx512f") {
+            let mut tr = aligned_input();
+            let sr = unsafe { tr.as_slice_mut_unchecked::<f16>() };
+            group.bench_function("avx512_f32roundtrip", |b| {
+                b.iter(|| <$roundtrip>::run(sr, ($($param)?)))
+            });
+        }
+        if std::is_x86_feature_detected!("avx512fp16") {
+            let mut tn = aligned_input();
+            let sn = unsafe { tn.as_slice_mut_unchecked::<f16>() };
+            group.bench_function("avx512fp16_native", |b| {
+                b.iter(|| <$native>::run(sn, ($($param)?)))
+            });
+        }
+        group.finish();
+    }};
+}
+
+fn benches(c: &mut Criterion) {
+    bench_triple!(
+        c,
+        "hardswish_f16",
+        tract_linalg::generic::hardswish::HHardSwish8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_hardswish_f16_64n,
+        tract_linalg::x86_64_fma::act_f16_fp16::x86_64_avx512fp16_hardswish_f16_128n
+    );
+    bench_triple!(
+        c,
+        "leaky_relu_f16",
+        tract_linalg::generic::leaky_relu::HLeakyRelu8,
+        tract_linalg::x86_64_fma::act_f16::x86_64_avx512_leaky_relu_f16_64n,
+        tract_linalg::x86_64_fma::act_f16_fp16::x86_64_avx512fp16_leaky_relu_f16_128n,
+        f16::from_f32(0.1)
+    );
+}
+
+criterion_group!(g, benches);
+criterion_main!(g);
@@ -0,0 +1,191 @@
+#![feature(asm)]
+#![allow(
+    dead_code,
+    non_upper_case_globals,
+    unused_macros,
+    non_snake_case,
+    unused_assignments
+)]
+
+use std::time::Instant;
+
+macro_rules! r2 { ($($stat:stmt)*) => { $( $stat )* $( $stat )* } }
+macro_rules! r4 { ($($stat:stmt)*) => { r2!(r2!($($stat)*)) }}
+macro_rules! r8 { ($($stat:stmt)*) => { r4!(r2!($($stat)*)) }}
+macro_rules! r16 { ($($stat:stmt)*) => { r4!(r4!($($stat)*)) }}
+macro_rules! r32 { ($($stat:stmt)*) => { r8!(r4!($($stat)*)) }}
+macro_rules! r64 { ($($stat:stmt)*) => { r8!(r8!($($stat)*)) }}
+macro_rules! r128 { ($($stat:stmt)*) => { r8!(r16!($($stat)*)) }}
+macro_rules! r1024 { ($($stat:stmt)*) => { r8!(r128!($($stat)*)) }}
+macro_rules! r4096 { ($($stat:stmt)*) => { r4!(r1024!($($stat)*)) }}
+
+const _F32: [f32; 1024] = [12.; 1024];
+const F32: *const f32 = _F32.as_ptr();
+
+/*
+fn ruin_cache() {
+let _a = (0..1000000).collect::<Vec<i32>>();
+}
+*/
+
+macro_rules! b {
+    ($f: block, $inner_loop: expr, $measures: expr) => {{
+        let mut values = Vec::with_capacity($measures);
+        for _ in 0..$measures {
+            //       ruin_cache();
+            let start = Instant::now();
+            for _ in 0..$inner_loop {
+                unsafe { $f };
+            }
+            values.push(start.elapsed());
+        }
+        values.sort();
+        values[$measures / 2].as_nanos() as f64 / 1e9 / $inner_loop as f64
+    }};
+}
+
+fn main() {
+    let cycle = b!(
+        {
+            r1024!(asm!("orr r0, r0, r0", out("r0") _));
+        },
+        1000,
+        1000
+    ) / 1024.;
+    let indep_fmla = b!(
+        {
+            r8!(asm!("
+                vmla.f32 q0, q0, q0
+                vmla.f32 q1, q1, q1
+                vmla.f32 q2, q2, q2
+                vmla.f32 q3, q3, q3
+                vmla.f32 q4, q4, q4
+                vmla.f32 q5, q5, q5
+                vmla.f32 q6, q6, q6
+                vmla.f32 q7, q7, q7
+                 ", out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _));
+        },
+        1000,
+        1000
+    ) / 64.;
+    eprintln!("rcp tp: indep fmla: {}", indep_fmla / cycle);
+    let dep_accu_fmla = b!(
+        {
+            r16!(asm!("
+                vmla.f32 q15, q0, q0
+                vmla.f32 q15, q1, q1
+                vmla.f32 q15, q2, q2
+                vmla.f32 q15, q3, q3
+                vmla.f32 q15, q4, q4
+                vmla.f32 q15, q5, q5
+                vmla.f32 q15, q6, q6
+                vmla.f32 q15, q7, q7
+                vmla.f32 q15, q8, q8
+                vmla.f32 q15, q9, q9
+                vmla.f32 q15, q10, q10
+                vmla.f32 q15, q11, q11
+                vmla.f32 q15, q12, q12
+                vmla.f32 q15, q13, q13
+                vmla.f32 q15, q14, q14
+                 ", out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
+                 out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
+        },
+        1000,
+        1000
+    ) / 16.
+        / 15.;
+    eprintln!("rcp tp: accu-dep fmla: {}", dep_accu_fmla / cycle);
+    let load_s_using_vld1_64 = b!(
+        {
+            let mut p = F32;
+            r16!(asm!("
+                vld1.64         {{d0-d3}}, [{0}]!
+                vld1.64         {{d4-d7}}, [{0}]!
+                vld1.64         {{d8-d11}}, [{0}]!
+                vld1.64         {{d12-d15}}, [{0}]!
+                vld1.64         {{d16-d19}}, [{0}]!
+                vld1.64         {{d20-d23}}, [{0}]!
+                vld1.64         {{d24-d27}}, [{0}]!
+                vld1.64         {{d28-d31}}, [{0}]!
+                 ", 
+                 inout(reg) p,
+                 out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
+                 out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
+        },
+        1000,
+        1000
+    ) / 16.
+        / 64.; // each line load 8 s
+    eprintln!(
+        "rcp tp: load s using vld1_64 ia {}",
+        load_s_using_vld1_64 / cycle
+    );
+    let load_s_using_vldm_q = b!(
+        {
+            let mut p = F32;
+            r16!(asm!("
+                vldm            {0}!, {{q0-q3}}
+                vldm            {0}!, {{q4-q7}}
+                vldm            {0}!, {{q8-q11}}
+                vldm            {0}!, {{q12-q15}}
+                 ", 
+                 inout(reg) p,
+                 out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
+                 out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
+        },
+        1000,
+        1000
+    ) / 16.
+        / 64.;
+    eprintln!(
+        "rcp tp: load s using vldmia q: {}",
+        load_s_using_vldm_q / cycle
+    );
+    let load = b!(
+        {
+            let mut p = F32;
+            r16!(asm!("
+                vldr.64  d0, [{0}]
+                vldr.64  d1, [{0}, #8]
+                vldr.64  d2, [{0}, #16]
+                vldr.64  d3, [{0}, #24]
+                vldr.64  d4, [{0}, #32]
+                vldr.64  d5, [{0}, #40]
+                vldr.64  d6, [{0}, #48]
+                vldr.64  d7, [{0}, #56]
+                vldr.64  d8, [{0}, #64]
+                vldr.64  d9, [{0}, #72]
+                vldr.64  d10, [{0}, #80]
+                vldr.64  d11, [{0}, #88]
+                vldr.64  d12, [{0}, #96]
+                vldr.64  d13, [{0}, #104]
+                vldr.64  d14, [{0}, #112]
+                vldr.64  d15, [{0}, #120]
+                vldr.64  d16, [{0}, #128]
+                vldr.64  d17, [{0}, #136]
+                vldr.64  d18, [{0}, #144]
+                vldr.64  d19, [{0}, #152]
+                vldr.64  d20, [{0}, #160]
+                vldr.64  d21, [{0}, #168]
+                vldr.64  d22, [{0}, #176]
+                vldr.64  d23, [{0}, #184]
+                vldr.64  d24, [{0}, #192]
+                vldr.64  d25, [{0}, #200]
+                vldr.64  d26, [{0}, #208]
+                vldr.64  d27, [{0}, #216]
+                vldr.64  d28, [{0}, #224]
+                vldr.64  d29, [{0}, #232]
+                vldr.64  d30, [{0}, #240]
+                vldr.64  d31, [{0}, #248]
+                add {0}, #256
+                 ", 
+                 inout(reg) p,
+                 out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
+                 out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
+        },
+        1000,
+        1000
+    ) / 16.
+        / 64.;
+    eprintln!("rcp tp: load s using vldr d + imm: {}", load / cycle);
+}
@@ -0,0 +1,87 @@
+use std::time::Instant;
+
+use tract_data::prelude::*;
+use tract_linalg::LADatum;
+use tract_linalg::frame::mmm::FusedSpec;
+use tract_linalg::frame::mmm::MatMatMulKer;
+
+fn ruin_cache() {
+    let _a = (0..1000000).collect::<Vec<i32>>();
+}
+
+fn bench_to_nanos<T: LADatum + Copy + num_traits::Zero, K: MatMatMulKer<T>>(
+    k: usize,
+    loops: usize,
+) -> f64 {
+    let item_size = T::datum_type().size_of();
+    let a = Tensor::zero_aligned::<T>(
+        &[(k + K::end_padding_packed_a()) * K::mr()],
+        K::alignment_bytes_packed_a(),
+    )
+    .unwrap();
+    let b = Tensor::zero_aligned::<T>(
+        &[(k + K::end_padding_packed_b()) * K::nr()],
+        K::alignment_bytes_packed_b(),
+    )
+    .unwrap();
+    let mut c = Tensor::zero::<T>(&[K::mr() * K::nr()]).unwrap();
+    let ref a = InputStoreKer::Packed {
+        ptr: unsafe { a.as_ptr_unchecked::<u8>() as _ },
+    };
+    let ref b = InputStoreKer::Packed {
+        ptr: unsafe { b.as_ptr_unchecked::<u8>() as _ },
+    };
+    let ref c = OutputStoreKer {
+        ptr: unsafe { c.as_ptr_mut_unchecked::<u8>() as _ },
+        item_size,
+        col_byte_stride: (item_size * K::mr()) as isize,
+        row_byte_stride: item_size as isize,
+    };
+    let ref linear = LinearSpec::Mul { k };
+    let op = MatMatMulKerSpec {
+        a,
+        b,
+        c,
+        linear,
+        non_linear: std::ptr::null(),
+    };
+    let mut values = Vec::with_capacity(loops);
+    for _ in 0..loops {
+        ruin_cache();
+        let start = Instant::now();
+        K::kernel(&op);
+        values.push(start.elapsed());
+    }
+    values.sort();
+    values[loops / 2].as_nanos() as f64
+}
+
+fn model<T: Datum + Copy + num_traits::Zero, K: MatMatMulKer<T>>() -> (f64, f64) {
+    let x = 1000;
+    let zp = bench_to_nanos::<T, K>(0, 10000);
+    let y = bench_to_nanos::<T, K>(x, 1000);
+    let slope = (y - zp) / x as f64;
+    (slope, zp)
+}
+
+fn as_match_line<T: Datum + Copy + num_traits::Zero, K: MatMatMulKer<T>>() {
+    let coeffs = model::<T, K>();
+    println!(
+        "({:?}, {}, {}) => {} * k + {},",
+        K::name(),
+        K::mr(),
+        K::nr(),
+        (coeffs.0 * 1000.).round(),
+        (coeffs.1 * 1000.).round()
+    );
+}
+
+fn main() {
+    use tract_linalg::arm64::*;
+    as_match_line::<f32, MatMatMulF32x16x4>();
+    as_match_line::<f32, MatMatMulF32x12x8>();
+    as_match_line::<f32, MatMatMulF32x8x8>();
+    as_match_line::<f32, MatMatMulF32x16x4A53>();
+    as_match_line::<f32, MatMatMulF32x12x8A53>();
+    as_match_line::<f32, MatMatMulF32x8x8A53>();
+}
@@ -0,0 +1,72 @@
+#![allow(dead_code)]
+
+use criterion::{Criterion, criterion_group, criterion_main};
+use tract_linalg::mmm::MatMatMul;
+
+#[path = "utils.rs"]
+mod utils;
+use utils::mat_mat_with_mm;
+
+fn run(c: &mut Criterion, name: &str, mmm: &dyn MatMatMul, m: usize, k: usize, n: usize) {
+    let mut group = c.benchmark_group(format!("avx512_zombie/{name}"));
+    let id = format!("{m}x{k}x{n}");
+    group.bench_with_input(
+        criterion::BenchmarkId::new("hot", &id),
+        &(tract_data::prelude::DatumType::F32, m, k, n, false),
+        |b, p| mat_mat_with_mm(b, mmm, p),
+    );
+    group.bench_with_input(
+        criterion::BenchmarkId::new("cold", &id),
+        &(tract_data::prelude::DatumType::F32, m, k, n, true),
+        |b, p| mat_mat_with_mm(b, mmm, p),
+    );
+}
+
+fn benches(c: &mut Criterion) {
+    if !std::is_x86_feature_detected!("avx512f") {
+        eprintln!("avx512f not available, skipping");
+        return;
+    }
+
+    use tract_data::prelude::DatumType::F32;
+    use tract_linalg::x86_64_fma::mmm::*;
+
+    // Representative large-K, square-ish M case.
+    let (m, k) = (64usize, 256usize);
+
+    // N = 5 : zombie was 32x5 vs old 64x3.
+    run(c, "N5_64x3_explicit", &*avx512_mmm_f32_64x3.mmm(), m, k, 5);
+    run(c, "N5_32x5_explicit", &*avx512_mmm_f32_32x5.mmm(), m, k, 5);
+
+    // N = 6 : zombie was 32x6 vs old 64x3.
+    run(c, "N6_64x3_explicit", &*avx512_mmm_f32_64x3.mmm(), m, k, 6);
+    run(c, "N6_32x6_explicit", &*avx512_mmm_f32_32x6.mmm(), m, k, 6);
+
+    // N = 8 : zombie was 16x8 vs old 48x4.
+    run(c, "N8_48x4_explicit", &*avx512_mmm_f32_48x4.mmm(), m, k, 8);
+    run(c, "N8_16x8_explicit", &*avx512_mmm_f32_16x8.mmm(), m, k, 8);
+
+    // What does the live dispatcher pick for these shapes? If the picker
+    // is healthy these match the zombie numbers above, kernel name printed
+    // to stderr at startup.
+    for n in [5usize, 6, 8] {
+        let mmm = tract_linalg::ops()
+            .mmm(F32, Some(m), Some(k), Some(n))
+            .unwrap();
+        eprintln!("dispatcher@m={m},k={k},n={n} picked {}", mmm.name());
+        run(c, &format!("N{n}_dispatch"), &*mmm, m, k, n);
+    }
+
+    // Trace-only: a few shapes where M-padding overhead with the old
+    // picker was high. We expect the M-aware picker to pick smaller-mr
+    // kernels here.
+    for (m, n) in [(20usize, 2), (33, 3), (50, 4), (17, 5), (1000, 64)] {
+        let mmm = tract_linalg::ops()
+            .mmm(F32, Some(m), Some(k), Some(n))
+            .unwrap();
+        eprintln!("dispatcher@m={m},k={k},n={n} picked {}", mmm.name());
+    }
+}
+
+criterion_group!(g, benches);
+criterion_main!(g);
@@ -0,0 +1,40 @@
+// Microbenchmark: AVX-512 (zmm, 16-wide) erf kernel vs the generic scalar
+// SErf4 (no FMA predecessor exists on x86). All buffers are 64-byte aligned
+// (AVX-512 alignment_bytes) and a multiple of 64 elements so the kernel's
+// nr() = 64 divides the length.
+
+use criterion::*;
+use tract_data::prelude::*;
+use tract_linalg::element_wise::ElementWiseKer;
+
+const N: usize = 1024;
+
+fn aligned_input() -> Tensor {
+    let mut t = unsafe { Tensor::uninitialized_aligned::<f32>(&[N], 64).unwrap() };
+    let s = unsafe { t.as_slice_mut_unchecked::<f32>() };
+    for (i, x) in s.iter_mut().enumerate() {
+        *x = (i as f32 / 10.0).sin() * 5.0;
+    }
+    t
+}
+
+fn erf_f32(c: &mut Criterion) {
+    let mut g = c.benchmark_group("erf_f32");
+    g.throughput(Throughput::Elements(N as u64));
+    let mut tp = aligned_input();
+    let sp = unsafe { tp.as_slice_mut_unchecked::<f32>() };
+    g.bench_function("generic", |b| {
+        b.iter(|| tract_linalg::generic::SErf4::run(sp, ()))
+    });
+    if std::is_x86_feature_detected!("avx512f") {
+        let mut ta = aligned_input();
+        let sa = unsafe { ta.as_slice_mut_unchecked::<f32>() };
+        g.bench_function("avx512", |b| {
+            b.iter(|| tract_linalg::x86_64_fma::erf::x86_64_avx512_erf_f32_64n::run(sa, ()))
+        });
+    }
+    g.finish();
+}
+
+criterion_group!(g, erf_f32);
+criterion_main!(g);
@@ -0,0 +1,44 @@
+use criterion::*;
+use tract_data::prelude::*;
+
+use tract_linalg::element_wise::ElementWiseKer;
+
+fn gelu_f32(c: &mut Criterion) {
+    let mut group = c.benchmark_group("gelu_f32");
+    group.throughput(Throughput::Elements(1024));
+    let mut input = unsafe { Tensor::uninitialized_aligned::<f32>(&[1024], 16).unwrap() };
+    let input = unsafe { input.as_slice_mut_unchecked::<f32>() };
+    for (i, x) in input.iter_mut().enumerate() {
+        *x = (i as f32 / 10.0).sin() * 5.0;
+    }
+    group.bench_function("rust_scalar", |b| b.iter(|| rust_scalar(input)));
+    group.bench_function("linalg", |b| b.iter(|| linalg(input)));
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("linalg-asm-compose", |b| {
+        b.iter(|| tract_linalg::arm64::arm64simd_gelu_f32_4n::run(input, ()))
+    });
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("linalg-asm-fused", |b| {
+        b.iter(|| tract_linalg::arm64::arm64simd_gelu_f32_4n_fused::run(input, ()))
+    });
+}
+
+#[inline(never)]
+fn rust_scalar(input: &mut [f32]) {
+    // Match tract's GeluApproximate scalar formula (pow=3).
+    const SQRT_2_OVER_PI: f32 = 0.7978845608028654;
+    const COEF: f32 = 0.044715;
+    for x in input {
+        let v = *x;
+        let inner = SQRT_2_OVER_PI * (v + COEF * v * v * v);
+        *x = 0.5 * v * (1.0 + inner.tanh());
+    }
+}
+
+#[inline(never)]
+fn linalg(input: &mut [f32]) {
+    (tract_linalg::ops().gelu_f32)().run(input).unwrap();
+}
+
+criterion_group!(benches, gelu_f32);
+criterion_main!(benches);
@@ -0,0 +1,34 @@
+use criterion::*;
+use tract_data::prelude::*;
+
+use tract_linalg::element_wise::ElementWiseKer;
+
+fn hardswish_f32(c: &mut Criterion) {
+    let mut group = c.benchmark_group("hardswish_f32");
+    group.throughput(Throughput::Elements(1024));
+    let mut input = unsafe { Tensor::uninitialized_aligned::<f32>(&[1024], 16).unwrap() };
+    let input = unsafe { input.as_slice_mut_unchecked::<f32>() };
+    group.bench_function("rust", |b| b.iter(|| rust_f32(input)));
+    group.bench_function("linalg", |b| b.iter(|| linalg32(input)));
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("linalg-asm", |b| {
+        b.iter(|| tract_linalg::arm64::arm64simd_hardswish_f32_8n::run(input, ()))
+    });
+}
+
+#[inline(never)]
+fn rust_f32(input: &mut [f32]) {
+    const INV6: f32 = 1.0 / 6.0;
+    for x in input {
+        let relu6 = ((*x + 3.0).min(6.0)).max(0.0);
+        *x = *x * relu6 * INV6;
+    }
+}
+
+#[inline(never)]
+fn linalg32(input: &mut [f32]) {
+    (tract_linalg::ops().hardswish_f32)().run(input).unwrap();
+}
+
+criterion_group!(benches, hardswish_f32);
+criterion_main!(benches);
@@ -0,0 +1,215 @@
+#![allow(dead_code)]
+use std::time::Instant;
+
+use tract_data::prelude::*;
+use tract_linalg::frame::mmm::*;
+
+fn ruin_cache() {
+    // return;
+    let _a = (0..1000000).collect::<Vec<i32>>();
+}
+
+pub fn reference<T, K>(mr: usize, k: usize, nr: usize) -> Vec<f32>
+where
+    T: Datum + Copy + num_traits::Zero + tract_linalg::LADatum,
+    K: MatMatMulKer<T>,
+{
+    let mut vi = vec![0.0; k * nr];
+
+    for m in 0..mr {
+        for n in 0..nr {
+            for _ in 0..k {
+                let a: f32 = 1.0;
+                let b = 1.0;
+                let offset = { n + m * nr };
+                vi[offset] += a * b;
+            }
+        }
+    }
+    vi
+}
+
+fn bench_to_nanos<
+    T: Datum + Copy + num_traits::Zero + tract_linalg::LADatum,
+    K: MatMatMulKer<T>,
+>(
+    loops: usize,
+    m: usize,
+    n: usize,
+    k: usize,
+) -> f64 {
+    let kernel = K::mmm();
+
+    let mut a = Tensor::zero_aligned::<T>(
+        &[(k + K::end_padding_packed_a()) * m],
+        K::alignment_bytes_packed_a(),
+    )
+    .unwrap();
+
+    let mut a_plain = a.try_as_plain_mut().unwrap();
+    let mut v = a_plain.to_array_view_mut::<f32>().unwrap();
+    v += 1.0;
+    drop(v);
+    drop(a_plain);
+    let mut b = Tensor::zero_aligned::<T>(
+        &[(k + K::end_padding_packed_b()) * n],
+        K::alignment_bytes_packed_b(),
+    )
+    .unwrap();
+
+    let mut b_plain = b.try_as_plain_mut().unwrap();
+    let mut v = b_plain.to_array_view_mut::<f32>().unwrap();
+    v += 1.0;
+    drop(v);
+    drop(b_plain);
+    let mut c = Tensor::zero::<T>(&[n, m]).unwrap();
+
+    let ops = unsafe {
+        [
+            FusedSpec::AddMatMul {
+                k,
+                a: kernel.a_packed(4, k).wrap(&a.view()),
+                b: kernel.b_packed(4, k).wrap(&b.view()),
+            },
+            // FusedSpec::AddUnicast(kernel.c_view(1, 0).wrap(&c.view_mut())),
+            FusedSpec::Store(kernel.c_view(1, 0).wrap(&c.view_mut())),
+        ]
+    };
+
+    let mut values = Vec::with_capacity(loops);
+
+    for _ in 0..loops {
+        ruin_cache();
+        let start = Instant::now();
+        unsafe { kernel.run(m, n, &ops).unwrap() };
+        values.push(start.elapsed());
+    }
+
+    eprintln!(
+        "{:?} -> {:?}",
+        values.first().unwrap(),
+        values.last().unwrap()
+    );
+
+    values.sort();
+    values[loops / 2].as_nanos() as f64
+}
+
+fn model<T: Datum + Copy + num_traits::Zero + tract_linalg::LADatum, K: MatMatMulKer<T>>()
+-> (f64, f64) {
+    let x = 1000;
+    let zp = bench_to_nanos::<T, K>(1000, K::mr() * 4, K::nr() * 4, 0);
+    let y = bench_to_nanos::<T, K>(1000, K::mr() * 4, K::nr() * 4, x);
+    let slope = (y - zp) / x as f64;
+    (slope, zp)
+}
+
+fn as_match_line<T: Datum + Copy + num_traits::Zero + tract_linalg::LADatum, K: MatMatMulKer<T>>() {
+    let coeffs = model::<T, K>();
+    println!(
+        "({:?}, {}, {}) => {} * k + {}",
+        K::name(),
+        K::mr(),
+        K::nr(),
+        (coeffs.0),
+        (coeffs.1),
+    );
+}
+
+fn main() {
+    let core_id = core_affinity::get_core_ids().unwrap()[0];
+    core_affinity::set_for_current(core_id);
+    // as_match_line::<f32, fma_mmm_f32_64x1>();
+    // as_match_line::<f32, avx512_mmm_f32_128x1>();
+    // as_match_line::<f32, avx512_mmm_f32_16x1>();
+    // as_match_line::<f32, fma_mmm_f32_40x2>();
+    // as_match_line::<f32, fma_mmm_f32_32x3>();
+    // as_match_line::<f32, fma_mmm_f32_24x4>();
+    // as_match_line::<f32, fma_mmm_f32_16x5>();
+    // as_match_line::<f32, fma_mmm_f32_16x6>();
+    // as_match_line::<f32, fma_mmm_f32_8x8>();
+
+    // mmv_perf_m();
+    mmm_perf_batch_size();
+}
+
+// for mmv
+fn mmv_perf_m() {
+    use tract_linalg::x86_64_fma::mmm::*;
+    let core_id = core_affinity::get_core_ids().unwrap()[0];
+    core_affinity::set_for_current(core_id);
+    fn bench<T: Datum + Copy + num_traits::Zero + tract_linalg::LADatum, K: MatMatMulKer<T>>(
+        m: usize,
+    ) {
+        let val = bench_to_nanos::<T, K>(1000, m, 1, 100) / (m * 100) as f64;
+        print!("{val}\t");
+    }
+
+    print!("N\t");
+    print!("fma_mmm_f32_64x1\t");
+    print!("avx512_mmm_f32_128x1\t");
+    print!("avx512_mmm_f32_16x1\t");
+    println!();
+    for n in 1..=128 {
+        eprintln!("{n}");
+        print!("{n}\t");
+        bench::<f32, fma_mmm_f32_64x1>(n);
+        bench::<f32, avx512_mmm_f32_128x1>(n);
+        bench::<f32, avx512_mmm_f32_16x1>(n);
+        println!();
+    }
+}
+
+// output a csv file with the perf of the kernels wrt batch size
+fn mmm_perf_batch_size() {
+    use tract_linalg::x86_64_fma::mmm::*;
+    let core_id = core_affinity::get_core_ids().unwrap()[0];
+    core_affinity::set_for_current(core_id);
+    fn bench<T: Datum + Copy + num_traits::Zero + tract_linalg::LADatum, K: MatMatMulKer<T>>(
+        n: usize,
+    ) {
+        let val =
+            bench_to_nanos::<T, K>(1000, K::mr() * 4, n, 100) / (K::mr() * 4 * 100 * n) as f64;
+        print!("{val}\t");
+    }
+
+    print!("N\t");
+    print!("fma_mmm_f32_8x8\t");
+    print!("fma_mmm_f32_16x6\t");
+    print!("fma_mmm_f32_16x5\t");
+    print!("fma_mmm_f32_24x4\t");
+    print!("fma_mmm_f32_32x3\t");
+    print!("fma_mmm_f32_40x2\t");
+    print!("fma_mmm_f32_64x1\t");
+    print!("avx512_mmm_f32_128x1\t");
+    print!("avx512_mmm_f32_16x1\t");
+    print!("avx512_mmm_f32_16x12\t");
+    print!("avx512_mmm_f32_16x8\t");
+    print!("avx512_mmm_f32_32x6\t");
+    print!("avx512_mmm_f32_32x5\t");
+    print!("avx512_mmm_f32_48x4\t");
+    print!("avx512_mmm_f32_64x3\t");
+    print!("avx512_mmm_f32_80x2\t");
+    println!();
+    for n in 1..=128 {
+        eprintln!("{n}");
+        print!("{n}\t");
+        bench::<f32, fma_mmm_f32_8x8>(n);
+        bench::<f32, fma_mmm_f32_16x6>(n);
+        bench::<f32, fma_mmm_f32_16x5>(n);
+        bench::<f32, fma_mmm_f32_24x4>(n);
+        bench::<f32, fma_mmm_f32_32x3>(n);
+        bench::<f32, fma_mmm_f32_40x2>(n);
+        bench::<f32, fma_mmm_f32_64x1>(n);
+        bench::<f32, avx512_mmm_f32_128x1>(n);
+        bench::<f32, avx512_mmm_f32_16x1>(n);
+        bench::<f32, avx512_mmm_f32_16x12>(n);
+        bench::<f32, avx512_mmm_f32_16x8>(n);
+        bench::<f32, avx512_mmm_f32_32x6>(n);
+        bench::<f32, avx512_mmm_f32_32x5>(n);
+        bench::<f32, avx512_mmm_f32_48x4>(n);
+        bench::<f32, avx512_mmm_f32_64x3>(n);
+        bench::<f32, avx512_mmm_f32_80x2>(n);
+        println!();
+    }
+}
@@ -0,0 +1,72 @@
+use criterion::*;
+use tract_data::prelude::*;
+
+use tract_linalg::element_wise::ElementWiseKer;
+
+fn leaky_relu_f16(c: &mut Criterion) {
+    let mut group = c.benchmark_group("leaky_relu_f16");
+    group.throughput(Throughput::Elements(1024));
+    let mut input = unsafe { Tensor::uninitialized_aligned::<f16>(&[1024], 16).unwrap() };
+    let input = input.as_slice_mut::<f16>().unwrap();
+    let alpha = f16::from_f32(0.1);
+    group.bench_function("rust", |b| b.iter(|| rust_fp16(input, alpha)));
+    group.bench_function("rust_with_f16", |b| {
+        b.iter(|| unsafe { rust_with_fp16(input, alpha) })
+    });
+    group.bench_function("linalg", |b| b.iter(|| linalg16(input, alpha)));
+    group.bench_function("linalg-asm", |b| {
+        b.iter(|| tract_linalg::arm64::arm64fp16_leaky_relu_f16_16n::run(input, alpha))
+    });
+}
+
+#[inline(never)]
+fn rust_fp16(input: &mut [f16], alpha: f16) {
+    for x in input {
+        *x = if *x > f16::ZERO { *x } else { *x * alpha }
+    }
+}
+
+#[target_feature(enable = "fp16")]
+#[inline(never)]
+unsafe fn rust_with_fp16(input: &mut [f16], alpha: f16) {
+    for x in input {
+        *x = if *x > f16::ZERO { *x } else { *x * alpha }
+    }
+}
+
+#[inline(never)]
+fn linalg16(input: &mut [f16], alpha: f16) {
+    (tract_linalg::ops().leaky_relu_f16)()
+        .run_with_params(input, alpha)
+        .unwrap();
+}
+
+fn leaky_relu_f32(c: &mut Criterion) {
+    let mut group = c.benchmark_group("leaky_relu_f32");
+    group.throughput(Throughput::Elements(1024));
+    let mut input = unsafe { Tensor::uninitialized_aligned::<f32>(&[1024], 16).unwrap() };
+    let input = input.as_slice_mut::<f32>().unwrap();
+    let alpha = 0.1f32;
+    group.bench_function("rust", |b| b.iter(|| rust_fp32(input, alpha)));
+    group.bench_function("linalg", |b| b.iter(|| linalg32(input, alpha)));
+    group.bench_function("linalg-asm", |b| {
+        b.iter(|| tract_linalg::arm64::arm64simd_leaky_relu_f32_8n::run(input, alpha))
+    });
+}
+
+#[inline(never)]
+fn rust_fp32(input: &mut [f32], alpha: f32) {
+    for x in input {
+        *x = if *x > 0.0 { *x } else { *x * alpha }
+    }
+}
+
+#[inline(never)]
+fn linalg32(input: &mut [f32], alpha: f32) {
+    (tract_linalg::ops().leaky_relu_f32)()
+        .run_with_params(input, alpha)
+        .unwrap();
+}
+
+criterion_group!(benches, leaky_relu_f32, leaky_relu_f16);
+criterion_main!(benches);
@@ -0,0 +1,48 @@
+use criterion::*;
+use tract_data::internal::*;
+use tract_linalg::mmm::{AsInputValue, FusedSpec};
+
+use DatumType::F32;
+
+fn mat_vec_mul(c: &mut Criterion) {
+    let mut group = c.benchmark_group("mat_vec_mul");
+    unsafe {
+        {
+            let (m, k) = &(768usize, 256usize);
+            group.throughput(Throughput::Elements((m * k) as u64));
+            group.bench_with_input(
+                BenchmarkId::from_parameter(format!("{m}x{k}")),
+                &(m, k),
+                |be, &(&m, &k)| {
+                    let mmm = tract_linalg::ops()
+                        .mmm(F32, Some(m), Some(k), Some(1))
+                        .unwrap();
+                    let packing = &mmm.packings()[0];
+                    let a = Tensor::zero::<f32>(&[m, k]).unwrap();
+                    let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+                    let b = Tensor::zero::<f32>(&[k, 1]).unwrap();
+                    let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+                    let mut c = Tensor::zero::<f32>(&[m]).unwrap();
+                    be.iter(move || {
+                        mmm.run(
+                            m,
+                            1,
+                            &[
+                                FusedSpec::AddMatMul {
+                                    a: AsInputValue::Borrowed(&*pa),
+                                    b: AsInputValue::Borrowed(&*pb),
+                                    packing: 0,
+                                },
+                                FusedSpec::Store(mmm.c_view(Some(0), Some(0)).wrap(&c.view_mut())),
+                            ],
+                        )
+                    });
+                },
+            );
+        }
+    }
+    group.finish();
+}
+
+criterion_group!(benches, mat_vec_mul);
+criterion_main!(benches);
@@ -0,0 +1,37 @@
+use criterion::*;
+
+mod utils;
+use utils::*;
+
+fn all(c: &mut Criterion) {
+    // packed_packed: co, ci, n
+    //    direct_conv(c, "asr_2M", 24, 5, 40, 200, 1); // lda
+    packed_packed(c, "asr_2M", 256, 200, 24); // tdnn1
+    //    direct_conv(c, "asr_2M", 24, 3, 256, 256, 1); // tdnn2
+    //    direct_conv(c, "asr_2M", 24, 3, 256, 256, 3); // tdnn3
+    packed_packed(c, "asr_2M", 256, 256, 8); // fastlstm1 and 2 (input) x 8 (4 prod x 2 layers)
+    packed_packed(c, "asr_2M", 256, 128, 1); // fastlstm1 and 2 (hidden) x 64 (4 prod x 2 layers x 8 loops)
+    packed_packed(c, "asr_2M", 256, 256, 1); // fastlstm1 and 2 (rp) x 16 (2 layers x 8 loops)
+    //    direct_conv(c, "asr_2M", 8, 3, 256, 256, 1); // tdnn4, tdd5 (x2)
+    packed_packed(c, "asr_2M", 1690, 256, 8); // output
+
+    // 8M
+    packed_packed(c, "asr_8M", 512, 200, 24); // tdnn1
+    packed_packed(c, "asr_8M", 512, 512, 24); // tdnn2
+    packed_packed(c, "asr_8M", 512, 256, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
+    packed_vec(c, "asr_8M", 512, 256, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
+
+    // pseudo 15M
+    packed_packed(c, "asr_pseudo15M", 768, 200, 24); // tdnn1
+    packed_packed(c, "asr_pseudo15M", 768, 2304, 24); // tdnn2
+    packed_packed(c, "asr_pseudo15M", 768, 2304, 8); // tdnn3,4,5
+    packed_packed(c, "asr_pseudo15M", 768, 768, 8); // fastlstm1 and 2 (four parts, rec mat*mat)
+    packed_packed(c, "asr_pseudo15M", 768, 384, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
+    packed_vec(c, "asr_pseudo15M", 768, 384, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
+
+    // 15M
+    packed_vec(c, "asr_15M", 768, 256, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
+}
+
+criterion_group!(benches, all);
+criterion_main!(benches);
@@ -0,0 +1,47 @@
+extern crate criterion;
+use criterion::*;
+use tract_data::internal::*;
+use tract_linalg::mmm::{AsInputValue, FusedSpec};
+
+use DatumType::F32;
+
+fn mat_mul_smmm(be: &mut criterion::Bencher, &(m, k, n): &(usize, usize, usize)) {
+    unsafe {
+        let mmm = tract_linalg::ops()
+            .mmm(F32, Some(m), Some(k), Some(n))
+            .unwrap();
+        let a = Tensor::zero::<f32>(&[m, k]).unwrap();
+        let b = Tensor::zero::<f32>(&[k, n]).unwrap();
+        let packing = &mmm.packings()[0];
+        let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+        let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+
+        let mut c = Tensor::zero::<f32>(&[m, n]).unwrap();
+        be.iter(move || {
+            mmm.run(
+                m,
+                n,
+                &[
+                    FusedSpec::AddMatMul {
+                        a: AsInputValue::Borrowed(&*pa),
+                        b: AsInputValue::Borrowed(&*pb),
+                        packing: 0,
+                    },
+                    FusedSpec::Store(mmm.c_view(Some(0), Some(1)).wrap(&c.view_mut())),
+                ],
+            )
+        });
+    }
+}
+
+fn mat_mul_prepacked(c: &mut Criterion, m: usize, k: usize, n: usize) {
+    let mut group = c.benchmark_group("mat_mul_prepacked");
+    group.bench_function("smmm", |be| mat_mul_smmm(be, &(m, k, n)));
+}
+
+fn s64x288x21609(c: &mut Criterion) {
+    mat_mul_prepacked(c, 64, 288, 21609)
+}
+
+criterion::criterion_group!(benches, s64x288x21609);
+criterion::criterion_main!(benches);
@@ -0,0 +1,12 @@
+use criterion::*;
+
+mod utils;
+use utils::*;
+
+fn s16x60x8(c: &mut Criterion) {
+    packed_packed(c, "wavenet", 32, 32, 8); // postproc
+    packed_packed(c, "wavenet", 16, 60, 8);
+}
+
+criterion_group!(benches, s16x60x8);
+criterion_main!(benches);
@@ -0,0 +1,58 @@
+// int8 -> i32 GEMM (qmmm_i32) microbench. A/B the SME SMOPA kernel vs the NEON
+// fallback by running twice: default (SME) vs TRACT_SME_DISABLE=1 (arm64simd 8x8).
+extern crate criterion;
+use criterion::*;
+use tract_data::internal::*;
+use tract_linalg::mmm::{AsInputValue, FusedSpec};
+
+use DatumType::I32;
+
+fn qmmm(be: &mut criterion::Bencher, &(m, k, n): &(usize, usize, usize)) {
+    unsafe {
+        let mmm = tract_linalg::ops()
+            .mmm(I32, Some(m), Some(k), Some(n))
+            .unwrap();
+        // packing index 1 == i8i8 for both sme_qmmm_i32_32x32 and arm64simd_mmm_i32_8x8.
+        let a = Tensor::zero::<i8>(&[m, k]).unwrap();
+        let b = Tensor::zero::<i8>(&[k, n]).unwrap();
+        let packing = &mmm.packings()[1];
+        let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+        let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+        let mut c = Tensor::zero::<i32>(&[m, n]).unwrap();
+        be.iter(move || {
+            mmm.run(
+                m,
+                n,
+                &[
+                    FusedSpec::AddMatMul {
+                        a: AsInputValue::Borrowed(&*pa),
+                        b: AsInputValue::Borrowed(&*pb),
+                        packing: 1,
+                    },
+                    FusedSpec::Store(mmm.c_view(Some(0), Some(1)).wrap(&c.view_mut())),
+                ],
+            )
+        });
+    }
+}
+
+fn bench(c: &mut Criterion) {
+    let mut g = c.benchmark_group("qmmm_i8");
+    g.sample_size(20);
+    for &shape in &[
+        (256usize, 256usize, 256usize),
+        (512, 512, 512),
+        (1024, 1024, 1024),
+        (128, 768, 768),
+        (384, 768, 768),
+        (64, 2048, 2048),
+    ] {
+        let (m, k, n) = shape;
+        g.throughput(Throughput::Elements((m * k * n) as u64));
+        g.bench_function(format!("{m}x{k}x{n}"), |be| qmmm(be, &shape));
+    }
+    g.finish();
+}
+
+criterion::criterion_group!(benches, bench);
+criterion::criterion_main!(benches);
@@ -0,0 +1,62 @@
+// Microbench: fused RmsNorm vs the 4-call composition that tract-core currently
+// uses (MeanOfSquares + Add + Rsqrt + Mul). The composition is reconstructed
+// inline here in the same shape as `core::ops::nn::rms_norm::RmsNorm::eval`
+// drives it. Both versions run on a 64-byte-aligned f32 row.
+
+use criterion::*;
+use tract_data::prelude::*;
+
+fn aligned_row(n: usize) -> Tensor {
+    let mut t = unsafe { Tensor::uninitialized_aligned::<f32>(&[n], 64).unwrap() };
+    let s = unsafe { t.as_slice_mut_unchecked::<f32>() };
+    for (i, x) in s.iter_mut().enumerate() {
+        *x = (i as f32 / 10.0).sin() * 5.0;
+    }
+    t
+}
+
+#[inline(never)]
+fn composed_rms_norm(buf: &mut [f32], eps: f32) {
+    // Same shape as tract-core's RmsNorm::eval: separate passes for sum-of-squares,
+    // mean, +eps, rsqrt, multiply — each writing/reading the row once.
+    let mut sum_sq = 0.0_f32;
+    for &x in buf.iter() {
+        sum_sq += x * x;
+    }
+    let mean_sq = sum_sq / buf.len() as f32;
+    let added = mean_sq + eps;
+    let inv_std = added.sqrt().recip();
+    for x in buf.iter_mut() {
+        *x *= inv_std;
+    }
+}
+
+fn rms_norm(c: &mut Criterion) {
+    for &n in &[1024usize, 2048, 4096] {
+        let id = format!("{n}");
+        let mut g = c.benchmark_group(format!("rms_norm_f32/{id}"));
+        g.throughput(Throughput::Elements(n as u64));
+        let mut t = aligned_row(n);
+        let s = unsafe { t.as_slice_mut_unchecked::<f32>() };
+        g.bench_function("composed", |b| b.iter(|| composed_rms_norm(s, 1e-5)));
+        g.bench_function("generic", |b| {
+            b.iter(|| tract_linalg::generic::rms_norm::rms_norm_f32(s, 1e-5))
+        });
+        #[cfg(target_arch = "x86_64")]
+        if std::is_x86_feature_detected!("avx512f") {
+            g.bench_function("avx512", |b| {
+                b.iter(|| tract_linalg::x86_64_fma::rms_norm::rms_norm_f32(s, 1e-5))
+            });
+        }
+        #[cfg(target_arch = "aarch64")]
+        {
+            g.bench_function("neon", |b| {
+                b.iter(|| tract_linalg::arm64::arm64simd_rms_norm_f32(s, 1e-5))
+            });
+        }
+        g.finish();
+    }
+}
+
+criterion_group!(g, rms_norm);
+criterion_main!(g);
@@ -0,0 +1,34 @@
+#[macro_use]
+extern crate criterion;
+extern crate tract_linalg;
+use criterion::Criterion;
+
+fn ssigmoid(c: &mut Criterion, n: usize) {
+    c.bench_function(&format!("ssigmoid_tract_{n}"), move |be| {
+        let mut s = (0..n).map(|i| i as f32 / 10.0).collect::<Vec<f32>>();
+        let op = &(tract_linalg::ops().sigmoid_f32)();
+        be.iter(|| op.run(&mut s));
+    });
+}
+
+#[inline(never)]
+fn rust_sigmoid(x: &mut [f32]) {
+    for v in x {
+        *v = 1.0 / (1.0 + (-*v).exp());
+    }
+}
+
+fn ssigmoid_scalar(c: &mut Criterion, n: usize) {
+    c.bench_function(&format!("ssigmoid_scalar_{n}"), move |be| {
+        let mut s = (0..n).map(|i| i as f32 / 10.0).collect::<Vec<f32>>();
+        be.iter(|| rust_sigmoid(&mut s));
+    });
+}
+
+fn bs(c: &mut Criterion) {
+    ssigmoid(c, 1024);
+    ssigmoid_scalar(c, 1024);
+}
+
+criterion_group!(benches, bs);
+criterion_main!(benches);
@@ -0,0 +1,40 @@
+use criterion::*;
+use tract_data::prelude::*;
+
+use tract_linalg::element_wise::ElementWiseKer;
+
+fn silu_f32(c: &mut Criterion) {
+    let mut group = c.benchmark_group("silu_f32");
+    group.throughput(Throughput::Elements(1024));
+    let mut input = unsafe { Tensor::uninitialized_aligned::<f32>(&[1024], 16).unwrap() };
+    let input = unsafe { input.as_slice_mut_unchecked::<f32>() };
+    for (i, x) in input.iter_mut().enumerate() {
+        *x = (i as f32 / 10.0).sin() * 5.0;
+    }
+    group.bench_function("rust_scalar", |b| b.iter(|| rust_scalar(input)));
+    group.bench_function("linalg", |b| b.iter(|| linalg(input)));
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("linalg-asm-compose", |b| {
+        b.iter(|| tract_linalg::arm64::arm64simd_silu_f32_4n::run(input, ()))
+    });
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("linalg-asm-fused", |b| {
+        b.iter(|| tract_linalg::arm64::arm64simd_silu_f32_4n_fused::run(input, ()))
+    });
+}
+
+#[inline(never)]
+fn rust_scalar(input: &mut [f32]) {
+    for x in input {
+        let sigmoid = 1.0 / (1.0 + (-*x).exp());
+        *x = *x * sigmoid;
+    }
+}
+
+#[inline(never)]
+fn linalg(input: &mut [f32]) {
+    (tract_linalg::ops().silu_f32)().run(input).unwrap();
+}
+
+criterion_group!(benches, silu_f32);
+criterion_main!(benches);
@@ -0,0 +1,170 @@
+use criterion::*;
+use tract_data::prelude::*;
+use tract_linalg::element_wise::ElementWiseKer;
+use tract_linalg::generic::reduce::softmax_l2::{HSoftMaxL2, SSoftMaxL2};
+use tract_linalg::reduce::{MapReduceKer, ReduceKer};
+
+#[inline(never)]
+fn loop1_f32_naive(slice: &mut [f32]) -> f32 {
+    let mut max = f32::MIN;
+    for x in &*slice {
+        if *x > max {
+            max = *x;
+        }
+    }
+    max
+}
+
+#[inline(never)]
+fn loop2_f32(slice: &mut [f32], max: f32) -> f32 {
+    let mut sum = 0.;
+    for x in slice.iter_mut() {
+        *x = (*x - max).exp();
+        sum += *x;
+    }
+    sum
+}
+
+#[inline(never)]
+fn loop3_f32(slice: &mut [f32], sum: f32) {
+    let recip = sum.recip();
+    for x in slice {
+        *x *= recip;
+    }
+}
+
+#[inline(never)]
+fn rust_f32(slice: &mut [f32]) {
+    let max = loop1_f32_naive(slice);
+    let sum = loop2_f32(slice, max);
+    loop3_f32(slice, sum);
+}
+
+fn softmax_f32(c: &mut Criterion) {
+    let mut group = c.benchmark_group("softmax_f32");
+    // 1536 = 24*64 = 48*32: a multiple of both the FMA (32) and AVX-512 (64) tile
+    // widths, 64-byte aligned so both kernels run entirely on their fast aligned
+    // path (no prefix/suffix scalar fixup) for a fair before/after comparison.
+    group.throughput(Throughput::Elements(1536));
+    let mut input = unsafe { Tensor::uninitialized_aligned::<f32>(&[1536], 64).unwrap() };
+    let mut plain = input.try_as_plain_mut().unwrap();
+    let input = plain.as_slice_mut::<f32>().unwrap();
+    // Deterministic finite values so every kernel sees identical, well-behaved
+    // input (uninitialized memory could contain NaN/huge values that perturb the
+    // fast-compact-exp int conversion and skew the comparison).
+    for (i, x) in input.iter_mut().enumerate() {
+        *x = ((i % 97) as f32) * 0.1 - 5.0;
+    }
+    group.bench_function("rust", |b| b.iter(|| rust_f32(input)));
+    group.bench_function("loop1/naive", |b| b.iter(|| loop1_f32_naive(input)));
+    group.bench_function("loop1/generic", |b| {
+        b.iter(|| tract_linalg::generic::reduce::max::SMax4::red().run(input))
+    });
+    #[cfg(target_arch = "x86_64")]
+    group.bench_function("loop1/iasm", |b| {
+        b.iter(|| {
+            tract_linalg::x86_64_fma::max::x86_64_fma_max_f32_32n::red()
+                .run(input)
+                .unwrap();
+        })
+    });
+    #[cfg(target_arch = "x86_64")]
+    if is_x86_feature_detected!("avx512f") {
+        group.bench_function("loop1/avx512", |b| {
+            b.iter(|| {
+                tract_linalg::x86_64_fma::max::x86_64_avx512_max_f32_64n::red()
+                    .run(input)
+                    .unwrap();
+            })
+        });
+    }
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("loop1/intr", |b| {
+        b.iter(|| {
+            tract_linalg::arm64::arm64simd_max_f32_16n::red()
+                .run(input)
+                .unwrap();
+        })
+    });
+    group.bench_function("loop2/naive", |b| b.iter(|| loop2_f32(input, 1.0)));
+    group.bench_function("loop2/generic", |b| {
+        b.iter(|| SSoftMaxL2::red().run_with_params(input, 10.))
+    });
+    #[cfg(target_arch = "x86_64")]
+    group.bench_function("loop2/iasm", |b| {
+        b.iter(|| {
+            tract_linalg::x86_64_fma::softmax::x86_64_fma_softmax2_fastcompact_f32_32n::red()
+                .run_with_params(input, 10.)
+                .unwrap()
+        });
+    });
+    #[cfg(target_arch = "x86_64")]
+    if is_x86_feature_detected!("avx512f") {
+        group.bench_function("loop2/avx512", |b| {
+            b.iter(|| {
+                tract_linalg::x86_64_fma::softmax::x86_64_avx512_softmax2_fastcompact_f32_64n::red()
+                    .run_with_params(input, 10.)
+                    .unwrap()
+            });
+        });
+    }
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("loop2/iasm", |b| {
+        b.iter(|| {
+            tract_linalg::arm64::arm64simd_softmax2_fastcompact_f32_16n::red()
+                .run_with_params(input, 0.21)
+                .unwrap()
+        });
+    });
+    group.bench_function("loop3/naive", |b| b.iter(|| loop3_f32(input, 0.21)));
+    group.bench_function("loop3/generic", |b| {
+        b.iter(|| {
+            tract_linalg::generic::by_scalar::SMulByScalar4::ew().run_with_params(input, 0.21)
+        })
+    });
+    #[cfg(target_arch = "x86_64")]
+    group.bench_function("loop3/iasm", |b| {
+        b.iter(|| {
+            tract_linalg::x86_64_fma::by_scalar::x86_64_avx_f32_mul_by_scalar_32n::ew()
+                .run_with_params(input, 0.21)
+                .unwrap()
+        });
+    });
+    #[cfg(target_arch = "aarch64")]
+    group.bench_function("loop3/iasm", |b| {
+        b.iter(|| {
+            tract_linalg::arm64::arm64simd_mul_by_scalar_f32_16n::ew()
+                .run_with_params(input, 0.21)
+                .unwrap()
+        });
+    });
+}
+
+fn softmax_f16(c: &mut Criterion) {
+    let mut group = c.benchmark_group("softmax_f16");
+    // 1536 = 64*24 (multiple of avx512 f16 nr=64 and generic h nr=8).
+    const N: usize = 1536;
+    group.throughput(Throughput::Elements(N as u64));
+    let mut input = unsafe { Tensor::uninitialized_aligned::<f16>(&[N], 64).unwrap() };
+    let mut plain = input.try_as_plain_mut().unwrap();
+    let input = plain.as_slice_mut::<f16>().unwrap();
+    for (i, x) in input.iter_mut().enumerate() {
+        *x = f16::from_f32((i as f32 / 10.0).sin() * 5.0);
+    }
+    group.bench_function("loop2/generic", |b| {
+        b.iter(|| HSoftMaxL2::red().run_with_params(input, f16::from_f32(10.0)))
+    });
+    #[cfg(target_arch = "x86_64")]
+    if std::is_x86_feature_detected!("avx512f") {
+        group.bench_function("loop2/avx512", |b| {
+            b.iter(|| {
+                tract_linalg::x86_64_fma::softmax::x86_64_avx512_softmax2_fastcompact_f16_64n::red()
+                    .run_with_params(input, f16::from_f32(10.0))
+                    .unwrap()
+            });
+        });
+    }
+}
+
+criterion_group!(benches, softmax_f32, softmax_f16);
+criterion_main!(benches);
@@ -0,0 +1,126 @@
+#![allow(dead_code)]
+use criterion::*;
+use tract_data::internal::*;
+use tract_linalg::mmm::{FusedSpec, MMMInputValue, MatMatMul};
+
+use DatumType::*;
+use tract_linalg::mmm::AsInputValue;
+
+pub fn packed_packed(c: &mut Criterion, name: &str, m: usize, k: usize, n: usize) {
+    let mut group = c.benchmark_group(format!("{name}/packed_packed"));
+    group.throughput(Throughput::Elements((m * k * n) as u64));
+    let id = format!("{m}x{k}x{n}");
+    group.bench_with_input(
+        BenchmarkId::new("f32/cold", &id),
+        &(F32, m, k, n, true),
+        mat_mat,
+    );
+    group.bench_with_input(
+        BenchmarkId::new("f32/hot", &id),
+        &(F32, m, k, n, false),
+        mat_mat,
+    );
+    group.bench_with_input(
+        BenchmarkId::new("i8/cold", &id),
+        &(I8, m, k, n, true),
+        mat_mat,
+    );
+    group.bench_with_input(
+        BenchmarkId::new("i8/hot", &id),
+        &(I8, m, k, n, false),
+        mat_mat,
+    );
+}
+
+pub fn packed_vec(c: &mut Criterion, name: &str, m: usize, k: usize, n: usize) {
+    assert_eq!(n, 1);
+    let mut group = c.benchmark_group(format!("{name}/packed_vec"));
+    group.throughput(Throughput::Elements((m * k * n) as u64));
+    let id = format!("{m}x{k}x{n}");
+    group.bench_with_input(
+        BenchmarkId::new("f32/cold", &id),
+        &(F32, m, k, n, true),
+        mat_mat,
+    );
+    group.bench_with_input(
+        BenchmarkId::new("f32/hot", &id),
+        &(F32, m, k, n, false),
+        mat_mat,
+    );
+    group.bench_with_input(
+        BenchmarkId::new("i8/cold", &id),
+        &(I8, m, k, n, true),
+        mat_mat,
+    );
+    group.bench_with_input(
+        BenchmarkId::new("i8/hot", &id),
+        &(I8, m, k, n, false),
+        mat_mat,
+    );
+}
+
+pub fn ruin_cache() {
+    let _a = (0..1000000).collect::<Vec<i32>>();
+}
+
+#[allow(clippy::too_many_arguments)]
+unsafe fn run(
+    m: usize,
+    _k: usize,
+    n: usize,
+    be: &mut Bencher,
+    mmm: &dyn MatMatMul,
+    a: &dyn MMMInputValue,
+    b: &dyn MMMInputValue,
+    cold: bool,
+) {
+    let mut scratch = unsafe { mmm.allocate_scratch_space() };
+    be.iter_custom(move |iters| {
+        let mut dur = std::time::Duration::default();
+        for _ in 0..iters {
+            if cold {
+                ruin_cache();
+            }
+            let instant = std::time::Instant::now();
+            unsafe {
+                mmm.run_with_scratch_space(
+                    m,
+                    n,
+                    scratch.as_mut(),
+                    &[FusedSpec::AddMatMul {
+                        a: AsInputValue::Borrowed(a),
+                        b: AsInputValue::Borrowed(b),
+                        packing: 0,
+                    }],
+                )
+                .unwrap()
+            };
+            let time = instant.elapsed();
+            dur += time;
+        }
+        dur
+    });
+}
+
+fn mat_mat(be: &mut Bencher, params: &(DatumType, usize, usize, usize, bool)) {
+    let (dt, m, k, n, _) = *params;
+    let mm = tract_linalg::ops()
+        .mmm(dt, Some(m), Some(k), Some(n))
+        .unwrap();
+    mat_mat_with_mm(be, &*mm, params)
+}
+
+pub fn mat_mat_with_mm(
+    be: &mut Bencher,
+    mmm: &dyn MatMatMul,
+    &(dt, m, k, n, cold): &(DatumType, usize, usize, usize, bool),
+) {
+    let a = Tensor::zero_dt(dt, &[m, k]).unwrap();
+    let b = Tensor::zero_dt(dt, &[k, n]).unwrap();
+    let packing = &mmm.packings()[0];
+    let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+    let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+    unsafe {
+        run(m, k, n, be, mmm, &*pa, &*pb, cold);
+    }
+}
@@ -0,0 +1,51 @@
+use criterion::measurement::WallTime;
+use criterion::*;
+use tract_data::internal::*;
+
+#[allow(dead_code)]
+#[path = "../tests/virtual_im2col.rs"]
+mod virtual_im2col;
+use virtual_im2col::ConvProblem;
+
+fn conv(
+    c: &mut BenchmarkGroup<WallTime>,
+    ci: usize,
+    h: usize,
+    w: usize,
+    co: usize,
+    kh: usize,
+    kw: usize,
+) {
+    // CHW HWIO
+    let input = Tensor::zero::<f32>(&[ci, h, w]).unwrap();
+    let filters = Tensor::zero::<f32>(&[kh, kw, ci, co]).unwrap();
+    let mut cv = ConvProblem {
+        input,
+        filters,
+        lazy_im2col: false,
+    };
+    c.bench_function("eager", |b| {
+        b.iter(|| {
+            cv.tract().unwrap();
+        })
+    });
+    cv.lazy_im2col = true;
+    c.bench_function("lazy", |b| {
+        b.iter(|| {
+            cv.tract().unwrap();
+        })
+    });
+}
+
+fn ex1(c: &mut Criterion) {
+    let mut c = c.benchmark_group("ex1");
+    conv(&mut c, 32, 256, 256, 32, 3, 3);
+}
+
+fn big(c: &mut Criterion) {
+    let mut c = c.benchmark_group("big");
+    conv(&mut c, 1, 1024, 1024, 99, 3, 3);
+}
+
+criterion_group!(benches, ex1, big);
+criterion_main!(benches);
@@ -0,0 +1,70 @@
+#![allow(dead_code)]
+// Kernel-level benchmark: AVX-512 VNNI int8 GEMM (avx512vnni_mmm_i32_8x8, VPDPBUSD
+// over the K=4-inner PackedI8K4 layout) vs the AVX2 int8 path (avx2_mmm_i32_8x8,
+// vpmaddubsw-style widening). Both run the i8i8 packing (index 1) over the same
+// M/K/N so the only difference is the matmul inner loop.
+use criterion::*;
+use tract_data::internal::*;
+use tract_linalg::mmm::{AsInputValue, FusedSpec, MatMatMul};
+
+fn run_kernel(be: &mut Bencher, mmm: &dyn MatMatMul, m: usize, k: usize, n: usize) {
+    let a = Tensor::zero_dt(DatumType::I8, &[m, k]).unwrap();
+    let b = Tensor::zero_dt(DatumType::I8, &[k, n]).unwrap();
+    let (pack_a, pack_b) = &mmm.packings()[1];
+    let pa = pack_a.prepare_one(&a, 1, 0).unwrap();
+    let pb = pack_b.prepare_one(&b, 0, 1).unwrap();
+    let mut scratch = unsafe { mmm.allocate_scratch_space() };
+    be.iter_custom(|iters| {
+        let mut dur = std::time::Duration::default();
+        for _ in 0..iters {
+            let t = std::time::Instant::now();
+            unsafe {
+                mmm.run_with_scratch_space(
+                    m,
+                    n,
+                    scratch.as_mut(),
+                    &[FusedSpec::AddMatMul {
+                        a: AsInputValue::Borrowed(&*pa),
+                        b: AsInputValue::Borrowed(&*pb),
+                        packing: 1,
+                    }],
+                )
+                .unwrap()
+            };
+            dur += t.elapsed();
+        }
+        dur
+    });
+}
+
+fn benches(c: &mut Criterion) {
+    if !std::is_x86_feature_detected!("avx512vnni") {
+        eprintln!("avx512vnni not available, skipping");
+        return;
+    }
+    use tract_linalg::x86_64_fma::mmm::*;
+    for &(m, k, n) in &[
+        (64usize, 256usize, 64usize),
+        (256, 256, 256),
+        (512, 512, 512),
+        (1024, 1024, 64),
+    ] {
+        let id = format!("{m}x{k}x{n}");
+        let mut g = c.benchmark_group("vnni_i32/packed_packed");
+        g.throughput(Throughput::Elements((m * k * n) as u64));
+        g.bench_with_input(
+            BenchmarkId::new("avx2", &id),
+            &(m, k, n),
+            |b, &(m, k, n)| run_kernel(b, &*avx2_mmm_i32_8x8.mmm(), m, k, n),
+        );
+        g.bench_with_input(
+            BenchmarkId::new("avx512vnni", &id),
+            &(m, k, n),
+            |b, &(m, k, n)| run_kernel(b, &*avx512vnni_mmm_i32_8x8.mmm(), m, k, n),
+        );
+        g.finish();
+    }
+}
+
+criterion_group!(g, benches);
+criterion_main!(g);
@@ -0,0 +1,704 @@
+//! WASM kernel microbenches. Run on wasm32 only.
+//!
+//!   RUSTFLAGS='-C target-feature=+simd128' \
+//!     CARGO_TARGET_WASM32_WASIP1_RUNNER='wasmtime --env RUST_TEST_NOCAPTURE=1 --' \
+//!     cargo bench --release --target wasm32-wasip1 -p tract-linalg --bench wasm
+//!
+//! Re-run with `+simd128,+relaxed-simd` to compare baseline mul+add against
+//! the FMA emit driven by the `madd_f32x4!` macro in `linalg/src/wasm.rs`.
+
+#[cfg(not(target_arch = "wasm32"))]
+fn main() {
+    eprintln!("this bench only runs on wasm32 targets — skipping on host");
+}
+
+#[cfg(target_arch = "wasm32")]
+fn main() {
+    let target = if cfg!(target_feature = "relaxed-simd") {
+        "+simd128,+relaxed-simd (FMA)"
+    } else {
+        "+simd128 only (mul+add)"
+    };
+
+    eprintln!("=== WASM 8x8 GEMM microbench ({target}) ===");
+    bench_8x8::run();
+
+    eprintln!();
+    eprintln!("=== Isolated 32x1 GEMV microbench ({target}) ===");
+    bench_32x1::run();
+
+    eprintln!();
+    eprintln!("=== Isolated 16x1 GEMV microbench ({target}) ===");
+    bench_16x1::run();
+
+    eprintln!();
+    eprintln!("=== int8 (i8->i32) 4x4 GEMM: wasm SIMD vs generic scalar ({target}) ===");
+    bench_i8_4x4::run();
+
+    #[cfg(target_feature = "relaxed-simd")]
+    {
+        eprintln!();
+        eprintln!("=== int8 relaxed-dot prototype: relaxed_dot vs widening (4x4 tile) ===");
+        bench_relaxed_dot::run();
+    }
+    #[cfg(not(target_feature = "relaxed-simd"))]
+    eprintln!("\n(int8 relaxed-dot prototype skipped — rebuild with +relaxed-simd)");
+}
+
+#[cfg(target_arch = "wasm32")]
+mod bench_8x8 {
+    //! Microbench: time `wasm_f32_8x8` (the GEMM kernel for N>=2) at shapes
+    //! relevant to DFN3, transformer FFN, and CNN→GEMM workloads.
+
+    use std::time::Instant;
+    use tract_data::internal::*;
+    use tract_linalg::mmm::{AsInputValue, FusedSpec};
+
+    fn run_one(
+        kernel: &dyn tract_linalg::mmm::MatMatMul,
+        m: usize,
+        k: usize,
+        n: usize,
+        iters: usize,
+    ) -> f64 {
+        let packing = &kernel.packings()[0];
+        let a = Tensor::zero::<f32>(&[m, k]).unwrap();
+        let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+        let b = Tensor::zero::<f32>(&[k, n]).unwrap();
+        let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+        let mut c = Tensor::zero::<f32>(&[m, n]).unwrap();
+
+        for _ in 0..50 {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        n,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: 0,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(1)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+
+        let t0 = Instant::now();
+        for _ in 0..iters {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        n,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: 0,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(1)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+        let elapsed = t0.elapsed();
+        elapsed.as_secs_f64() / iters as f64 * 1e9
+    }
+
+    fn pick(name: &str) -> Box<dyn tract_linalg::mmm::MatMatMul> {
+        let mut ops = tract_linalg::generic();
+        tract_linalg::wasm::plug(&mut ops);
+        for impl_ in ops.mmm_impls() {
+            if impl_.name() == name {
+                return impl_.clone();
+            }
+        }
+        panic!("kernel {name} not registered")
+    }
+
+    fn bench_shape(label: &str, m: usize, k: usize, n: usize, iters: usize) {
+        let k88 = pick("wasm_f32_8x8");
+        let ns = run_one(&*k88, m, k, n, iters);
+        let m_tiles = m.div_ceil(8);
+        let n_tiles = n.div_ceil(8);
+        let total_tiles = m_tiles * n_tiles;
+        let per_tile_ns = ns / total_tiles as f64;
+        eprintln!(
+            "{label} (m={m} k={k} n={n}, iters={iters}): {ns:.0} ns/call \
+             ({total_tiles} 8x8 tiles, {per_tile_ns:.1} ns/tile)"
+        );
+    }
+
+    pub fn run() {
+        // DFN3 N>1 GEMM case (the primary 8x8 hit on DFN3).
+        bench_shape("DFN3-style m=64 k=64 n=8", 64, 64, 8, 50_000);
+        // Larger N — typical batched/transformer GEMM.
+        bench_shape("m=64 k=64 n=64", 64, 64, 64, 10_000);
+        bench_shape("m=128 k=128 n=8", 128, 128, 8, 20_000);
+        bench_shape("m=128 k=128 n=64", 128, 128, 64, 5_000);
+        bench_shape("m=256 k=256 n=8", 256, 256, 8, 5_000);
+        bench_shape("m=256 k=256 n=64", 256, 256, 64, 1_000);
+        // Whisper-tiny FFN-ish (large K, small N).
+        bench_shape("m=384 k=1536 n=8", 384, 1536, 8, 1_000);
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+mod bench_32x1 {
+    //! Isolated, statistics-aware microbench for `wasm_f32_32x1` to investigate
+    //! the apparent regression at M=100/256 in `microbench_dispatch_gemv`. That
+    //! bench loops all 4 GEMV kernels back-to-back at every shape, biasing the
+    //! later-running kernel (32x1) with cache contention and thermal buildup.
+    //! This module benches 32x1 alone, with min-of-N reporting across
+    //! repetitions to expose variance honestly.
+
+    use std::time::Instant;
+    use tract_data::internal::*;
+    use tract_linalg::mmm::{AsInputValue, FusedSpec};
+
+    fn run_one(kernel: &dyn tract_linalg::mmm::MatMatMul, m: usize, k: usize, iters: usize) -> f64 {
+        let packing = &kernel.packings()[0];
+        let a = Tensor::zero::<f32>(&[m, k]).unwrap();
+        let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+        let b = Tensor::zero::<f32>(&[k, 1]).unwrap();
+        let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+        let mut c = Tensor::zero::<f32>(&[m, 1]).unwrap();
+
+        // Generous warmup — 200 calls primes the JIT and hot caches.
+        for _ in 0..200 {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        1,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: 0,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(0)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+
+        let t0 = Instant::now();
+        for _ in 0..iters {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        1,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: 0,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(0)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+        let elapsed = t0.elapsed();
+        elapsed.as_secs_f64() / iters as f64 * 1e9
+    }
+
+    fn pick(name: &str) -> Box<dyn tract_linalg::mmm::MatMatMul> {
+        let mut ops = tract_linalg::generic();
+        tract_linalg::wasm::plug(&mut ops);
+        for impl_ in ops.mmm_impls() {
+            if impl_.name() == name {
+                return impl_.clone();
+            }
+        }
+        panic!("kernel {name} not registered")
+    }
+
+    fn bench_min_of_n(label: &str, m: usize, k: usize, iters: usize, repetitions: usize) {
+        let kernel = pick("wasm_f32_32x1");
+        let mut samples: Vec<f64> = Vec::with_capacity(repetitions);
+        for _ in 0..repetitions {
+            samples.push(run_one(&*kernel, m, k, iters));
+        }
+        samples.sort_by(|a, b| a.partial_cmp(b).unwrap());
+        let min = samples[0];
+        let median = samples[samples.len() / 2];
+        let max = samples[samples.len() - 1];
+        let pct_spread = (max - min) / min * 100.0;
+        eprintln!(
+            "{label} (m={m} k={k}, {iters} iters × {repetitions} reps): \
+             min={min:.0} median={median:.0} max={max:.0} ns/call (spread {pct_spread:.0}%)"
+        );
+    }
+
+    pub fn run() {
+        // Suspect shapes from microbench_dispatch_gemv (apparent regression):
+        bench_min_of_n("M=100 k=256", 100, 256, 10_000, 10);
+        bench_min_of_n("M=256 k=256", 256, 256, 5_000, 10);
+        bench_min_of_n("M=256 k=512", 256, 512, 2_000, 10);
+        // Reference shapes (showed clean speedup before):
+        bench_min_of_n("M=24 k=256", 24, 256, 30_000, 10);
+        bench_min_of_n("M=64 k=96", 64, 96, 20_000, 10);
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+mod bench_16x1 {
+    //! Isolated 16x1 GEMV microbench — same methodology as bench_32x1.
+    //! 16x1 has 4 SIMD accumulators per K-step, which under +relaxed-simd
+    //! exposes the destructive-fmla accumulator recurrence (4-cycle latency
+    //! throttling throughput to 1 FMA/cycle even though Apple Silicon pipes
+    //! can do 4). Used to validate that the fix in linalg/src/wasm.rs (which
+    //! routes 16x1 through `madd_f32x4_nofma!` to use separate mul+add)
+    //! recovers the regression PR #2199 missed.
+    use std::time::Instant;
+    use tract_data::internal::*;
+    use tract_linalg::mmm::{AsInputValue, FusedSpec};
+
+    fn run_one(kernel: &dyn tract_linalg::mmm::MatMatMul, m: usize, k: usize, iters: usize) -> f64 {
+        let packing = &kernel.packings()[0];
+        let a = Tensor::zero::<f32>(&[m, k]).unwrap();
+        let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+        let b = Tensor::zero::<f32>(&[k, 1]).unwrap();
+        let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+        let mut c = Tensor::zero::<f32>(&[m, 1]).unwrap();
+
+        for _ in 0..200 {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        1,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: 0,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(0)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+
+        let t0 = Instant::now();
+        for _ in 0..iters {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        1,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: 0,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(0)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+        let elapsed = t0.elapsed();
+        elapsed.as_secs_f64() / iters as f64 * 1e9
+    }
+
+    fn pick(name: &str) -> Box<dyn tract_linalg::mmm::MatMatMul> {
+        let mut ops = tract_linalg::generic();
+        tract_linalg::wasm::plug(&mut ops);
+        for impl_ in ops.mmm_impls() {
+            if impl_.name() == name {
+                return impl_.clone();
+            }
+        }
+        panic!("kernel {name} not registered")
+    }
+
+    fn bench_min_of_n(label: &str, m: usize, k: usize, iters: usize, repetitions: usize) {
+        let kernel = pick("wasm_f32_16x1");
+        let mut samples: Vec<f64> = Vec::with_capacity(repetitions);
+        for _ in 0..repetitions {
+            samples.push(run_one(&*kernel, m, k, iters));
+        }
+        samples.sort_by(|a, b| a.partial_cmp(b).unwrap());
+        let min = samples[0];
+        let median = samples[samples.len() / 2];
+        let max = samples[samples.len() - 1];
+        let pct_spread = (max - min) / min * 100.0;
+        eprintln!(
+            "{label} (m={m} k={k}, {iters} iters × {repetitions} reps): \
+             min={min:.0} median={median:.0} max={max:.0} ns/call (spread {pct_spread:.0}%)"
+        );
+    }
+
+    pub fn run() {
+        // 16x1's natural band per plug()'s mmv_f32 closure: M ∈ 9..=16
+        bench_min_of_n("M=9 k=256", 9, 256, 30_000, 10);
+        bench_min_of_n("M=12 k=256", 12, 256, 30_000, 10);
+        bench_min_of_n("M=16 k=96", 16, 96, 30_000, 10);
+        bench_min_of_n("M=16 k=256", 16, 256, 20_000, 10);
+        bench_min_of_n("M=16 k=512", 16, 512, 10_000, 10);
+        bench_min_of_n("M=16 k=1024", 16, 1024, 5_000, 10);
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+mod bench_i8_4x4 {
+    //! int8 (i8->i32) GEMM microbench: the new SIMD `wasm_i32_4x4` vs the scalar
+    //! `generic_i32_4x4` fallback. Both kernels expose the *identical* i8i8
+    //! PackedI8K4 packing (packing index 1), the same 4x4 tile and i32
+    //! accumulator — so the ratio is a clean read on what the SIMD
+    //! widening-extmul AddMatMul buys over the generic scalar loop. min-of-N
+    //! reporting per kernel to keep the variance honest.
+
+    use std::time::Instant;
+    use tract_data::internal::*;
+    use tract_linalg::mmm::{AsInputValue, FusedSpec, MatMatMul};
+
+    // i8i8 packing slot is index 1 on both generic_i32_4x4 and wasm_i32_4x4.
+    const I8I8: usize = 1;
+
+    fn run_one(kernel: &dyn MatMatMul, m: usize, k: usize, n: usize, iters: usize) -> f64 {
+        let packing = &kernel.packings()[I8I8];
+        let a = Tensor::zero::<i8>(&[m, k]).unwrap();
+        let pa = packing.0.prepare_one(&a, 1, 0).unwrap();
+        let b = Tensor::zero::<i8>(&[k, n]).unwrap();
+        let pb = packing.1.prepare_one(&b, 0, 1).unwrap();
+        let mut c = Tensor::zero::<i32>(&[m, n]).unwrap();
+
+        // Warmup: prime the JIT and hot caches.
+        for _ in 0..50 {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        n,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: I8I8,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(1)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+
+        let t0 = Instant::now();
+        for _ in 0..iters {
+            unsafe {
+                kernel
+                    .run(
+                        m,
+                        n,
+                        &[
+                            FusedSpec::AddMatMul {
+                                a: AsInputValue::Borrowed(&*pa),
+                                b: AsInputValue::Borrowed(&*pb),
+                                packing: I8I8,
+                            },
+                            FusedSpec::Store(kernel.c_view(Some(0), Some(1)).wrap(&c.view_mut())),
+                        ],
+                    )
+                    .unwrap();
+            }
+        }
+        let elapsed = t0.elapsed();
+        elapsed.as_secs_f64() / iters as f64 * 1e9
+    }
+
+    fn pick(name: &str) -> Box<dyn MatMatMul> {
+        let mut ops = tract_linalg::generic();
+        tract_linalg::wasm::plug(&mut ops);
+        for impl_ in ops.mmm_impls() {
+            if impl_.name() == name {
+                return impl_.clone();
+            }
+        }
+        panic!("kernel {name} not registered")
+    }
+
+    fn min_of_n(
+        kernel: &dyn MatMatMul,
+        m: usize,
+        k: usize,
+        n: usize,
+        iters: usize,
+        reps: usize,
+    ) -> f64 {
+        let mut samples: Vec<f64> = (0..reps).map(|_| run_one(kernel, m, k, n, iters)).collect();
+        samples.sort_by(|a, b| a.partial_cmp(b).unwrap());
+        samples[0]
+    }
+
+    fn bench(label: &str, m: usize, k: usize, n: usize, iters: usize, reps: usize) {
+        let wasm = pick("wasm_i32_4x4");
+        let generic = pick("generic_i32_4x4");
+        let w = min_of_n(&*wasm, m, k, n, iters, reps);
+        let g = min_of_n(&*generic, m, k, n, iters, reps);
+        let tiles = m.div_ceil(4) * n.div_ceil(4);
+        eprintln!(
+            "{label} (m={m} k={k} n={n}, {iters} iters × {reps} reps): \
+             wasm={w:.0} generic={g:.0} ns/call  speedup={:.2}x  \
+             ({tiles} 4x4 tiles, wasm {:.1} ns/tile)",
+            g / w,
+            w / tiles as f64
+        );
+    }
+
+    pub fn run() {
+        // Square GEMMs across sizes (compute-bound, the SIMD path's home turf).
+        bench("square m=64 k=64 n=64", 64, 64, 64, 5_000, 8);
+        bench("square m=128 k=128 n=128", 128, 128, 128, 1_000, 8);
+        bench("square m=256 k=256 n=256", 256, 256, 256, 200, 8);
+        // Transformer-ish: large K, moderate M/N (MiniLM/FFN projections).
+        bench("m=128 k=384 n=384", 128, 384, 384, 500, 8);
+        bench("m=64 k=1536 n=64", 64, 1536, 64, 1_000, 8);
+        // CNN→GEMM (InceptionV1-style im2col), small N.
+        bench("m=256 k=256 n=16", 256, 256, 16, 2_000, 8);
+    }
+}
+
+// Prototype: int8 4x4 tile via `i32x4_relaxed_dot_i8x16_i7x16_add` (SDOT-analog,
+// 4 i8 MACs/lane, no widening) vs the deterministic widening path. Only compiles
+// under +relaxed-simd. Isolates a single cache-resident 4x4 tile so the ratio is
+// a pure instruction-density read. Includes a bit-exactness check on wasmtime.
+#[cfg(all(target_arch = "wasm32", target_feature = "relaxed-simd"))]
+mod bench_relaxed_dot {
+    use std::arch::wasm32::*;
+    use std::hint::black_box;
+    use std::time::Instant;
+
+    // Logical A is [4][k] row-major (a[m*k + ik]); logical B is [k][4] row-major
+    // (b[ik*4 + n]). Reference 4x4 = sum_ik A[m][ik] * B[ik][n].
+    fn reference_tile(a: &[i8], b: &[i8], k: usize) -> [i32; 16] {
+        let mut c = [0i32; 16];
+        for ik in 0..k {
+            for m in 0..4 {
+                for n in 0..4 {
+                    c[m * 4 + n] += a[m * k + ik] as i32 * b[ik * 4 + n] as i32;
+                }
+            }
+        }
+        c
+    }
+
+    // K-major A: out[ik*4 + m] = A[m][ik] (m inner) — what the widening kernel reads.
+    fn pack_a_kmajor(a: &[i8], k: usize) -> Vec<i8> {
+        let mut o = vec![0i8; k * 4];
+        for ik in 0..k {
+            for m in 0..4 {
+                o[ik * 4 + m] = a[m * k + ik];
+            }
+        }
+        o
+    }
+    // K-major B is exactly the logical [ik*4 + n] layout already.
+
+    // M-major A, K contiguous, K padded to mult of 4: out[m*kp + ik] = A[m][ik].
+    fn pack_a_mmajor(a: &[i8], k: usize) -> (Vec<i8>, usize) {
+        let kp = k.div_ceil(4) * 4;
+        let mut o = vec![0i8; 4 * kp];
+        for m in 0..4 {
+            for ik in 0..k {
+                o[m * kp + ik] = a[m * k + ik];
+            }
+        }
+        (o, kp)
+    }
+    // K=4-inner B: out[kb*16 + n*4 + kr] = B[4kb+kr][n] — the relaxed-dot layout.
+    fn pack_b_k4(b: &[i8], k: usize) -> Vec<i8> {
+        let kp = k.div_ceil(4) * 4;
+        let mut o = vec![0i8; kp * 4];
+        for kb in 0..kp / 4 {
+            for kr in 0..4 {
+                let kk = 4 * kb + kr;
+                if kk >= k {
+                    continue;
+                }
+                for n in 0..4 {
+                    o[kb * 16 + n * 4 + kr] = b[kk * 4 + n];
+                }
+            }
+        }
+        o
+    }
+
+    // Current deterministic approach: widen B to i32x4 per k, splat A, mul+add.
+    unsafe fn widening_tile(a_km: *const i8, b_km: *const i8, k: usize) -> [i32; 16] {
+        unsafe {
+            let mut acc = [i32x4_splat(0); 4];
+            for ik in 0..k {
+                let bw = v128_load32_zero(b_km.add(4 * ik) as *const u32);
+                let bw = i16x8_extend_low_i8x16(bw);
+                let bw = i32x4_extend_low_i16x8(bw);
+                let ar = a_km.add(4 * ik);
+                acc[0] = i32x4_add(acc[0], i32x4_mul(i32x4_splat(*ar.add(0) as i32), bw));
+                acc[1] = i32x4_add(acc[1], i32x4_mul(i32x4_splat(*ar.add(1) as i32), bw));
+                acc[2] = i32x4_add(acc[2], i32x4_mul(i32x4_splat(*ar.add(2) as i32), bw));
+                acc[3] = i32x4_add(acc[3], i32x4_mul(i32x4_splat(*ar.add(3) as i32), bw));
+            }
+            let mut c = [0i32; 16];
+            for m in 0..4 {
+                v128_store(c[m * 4..].as_mut_ptr() as *mut v128, acc[m]);
+            }
+            c
+        }
+    }
+
+    // Relaxed-dot: per 4-K block, one v128 B-load shared across 4 rows; each row
+    // broadcasts its 4 K-bytes and issues one relaxed_dot. 64 MACs in 4 dots.
+    unsafe fn relaxed_tile(apk: *const i8, bpk: *const i8, kp: usize) -> [i32; 16] {
+        unsafe {
+            let mut acc = [i32x4_splat(0); 4];
+            for kb in 0..kp / 4 {
+                let b_all = v128_load(bpk.add(kb * 16) as *const v128);
+                for m in 0..4 {
+                    let a4 = (apk.add(m * kp + kb * 4) as *const i32).read_unaligned();
+                    let a_m = i32x4_splat(a4);
+                    acc[m] = i32x4_relaxed_dot_i8x16_i7x16_add(a_m, b_all, acc[m]);
+                }
+            }
+            let mut c = [0i32; 16];
+            for m in 0..4 {
+                v128_store(c[m * 4..].as_mut_ptr() as *mut v128, acc[m]);
+            }
+            c
+        }
+    }
+
+    fn gen_data(k: usize, seed: i32, bits7: bool) -> Vec<i8> {
+        (0..k * 4)
+            .map(|i| {
+                let v = ((i as i32)
+                    .wrapping_mul(97)
+                    .wrapping_add(seed)
+                    .wrapping_mul(31))
+                    & 0xff;
+                let v = (v - 128) as i8; // full i8 range
+                if bits7 {
+                    (v as i32).clamp(-63, 63) as i8
+                } else {
+                    v
+                }
+            })
+            .collect()
+    }
+
+    fn check(label: &str, k: usize, b_bits7: bool) {
+        let a = gen_data(k, 1, false);
+        let b = gen_data(k, 7, b_bits7);
+        let reference = reference_tile(&a, &b, k);
+
+        let a_km = pack_a_kmajor(&a, k);
+        let w = unsafe { widening_tile(a_km.as_ptr(), b.as_ptr(), k) };
+        assert_eq!(w, reference, "widening_tile mismatch ({label})");
+
+        let (a_mm, kp) = pack_a_mmajor(&a, k);
+        let b_k4 = pack_b_k4(&b, k);
+        let r = unsafe { relaxed_tile(a_mm.as_ptr(), b_k4.as_ptr(), kp) };
+        let exact = r == reference;
+        eprintln!(
+            "  correctness {label} (k={k}, B={}): widening=exact  relaxed={}",
+            if b_bits7 { "7-bit" } else { "full-i8" },
+            if exact {
+                "EXACT"
+            } else {
+                "DIFFERS (non-deterministic intermediate)"
+            }
+        );
+        if b_bits7 {
+            assert!(exact, "relaxed_dot must be exact when B is 7-bit ({label})");
+        }
+    }
+
+    fn time_relaxed(apk: &[i8], bpk: &[i8], kp: usize, iters: usize) -> f64 {
+        let mut sink = 0i32;
+        for _ in 0..50 {
+            sink ^= unsafe { relaxed_tile(apk.as_ptr(), bpk.as_ptr(), kp) }[0];
+        }
+        let t0 = Instant::now();
+        for _ in 0..iters {
+            let c = unsafe {
+                relaxed_tile(
+                    black_box(apk).as_ptr(),
+                    black_box(bpk).as_ptr(),
+                    black_box(kp),
+                )
+            };
+            sink ^= c[5];
+        }
+        black_box(sink);
+        t0.elapsed().as_secs_f64() / iters as f64 * 1e9
+    }
+
+    fn time_widening(a_km: &[i8], b_km: &[i8], k: usize, iters: usize) -> f64 {
+        let mut sink = 0i32;
+        for _ in 0..50 {
+            sink ^= unsafe { widening_tile(a_km.as_ptr(), b_km.as_ptr(), k) }[0];
+        }
+        let t0 = Instant::now();
+        for _ in 0..iters {
+            let c = unsafe {
+                widening_tile(
+                    black_box(a_km).as_ptr(),
+                    black_box(b_km).as_ptr(),
+                    black_box(k),
+                )
+            };
+            sink ^= c[5];
+        }
+        black_box(sink);
+        t0.elapsed().as_secs_f64() / iters as f64 * 1e9
+    }
+
+    fn min_of_n(f: &mut dyn FnMut() -> f64, reps: usize) -> f64 {
+        let mut s: Vec<f64> = (0..reps).map(|_| f()).collect();
+        s.sort_by(|a, b| a.partial_cmp(b).unwrap());
+        s[0]
+    }
+
+    fn bench(k: usize, iters: usize, reps: usize) {
+        let a = gen_data(k, 1, false);
+        let b = gen_data(k, 7, false);
+        let a_km = pack_a_kmajor(&a, k);
+        let (a_mm, kp) = pack_a_mmajor(&a, k);
+        let b_k4 = pack_b_k4(&b, k);
+
+        let w = min_of_n(&mut || time_widening(&a_km, &b, k, iters), reps);
+        let r = min_of_n(&mut || time_relaxed(&a_mm, &b_k4, kp, iters), reps);
+        eprintln!(
+            "  4x4 tile k={k} ({iters} iters × {reps} reps): \
+             widening={w:.1} relaxed={r:.1} ns/call  speedup={:.2}x",
+            w / r
+        );
+    }
+
+    pub fn run() {
+        // Bit-exactness on wasmtime: full-i8 (engine-dependent intermediate) and
+        // 7-bit B (guaranteed no i16 overflow → deterministic on any engine).
+        check("k=64", 64, false);
+        check("k=64", 64, true);
+        check("k=260-padded", 260, false);
+        check("k=260-padded", 260, true);
+        eprintln!();
+        // Throughput: single cache-resident 4x4 tile across K depths.
+        bench(64, 200_000, 8);
+        bench(256, 50_000, 8);
+        bench(1024, 10_000, 8);
+        bench(1536, 8_000, 8);
+    }
+}
@@ -0,0 +1,554 @@
+#![allow(
+    dead_code,
+    non_upper_case_globals,
+    unused_macros,
+    non_snake_case,
+    unused_assignments
+)]
+
+use std::arch::asm;
+
+// mod nano;
+
+#[repr(C, align(64))]
+struct Floats([f32; 256 * 1024 * 64]);
+const _F32: Floats = Floats([12.; 256 * 1024 * 64]);
+const F32: *const f32 = (&_F32) as *const Floats as *const f32;
+
+lazy_static::lazy_static! {
+    static ref TICK: f64 = unsafe { b8192!(asm!("or rax, rax", out("rax") _)) };
+}
+
+macro_rules! kloop {
+    ($filter: expr, $geo: literal, $n: expr, $path: literal, $ww: expr, $u: expr, $arch: expr) => {
+        let label = $path.split("/").last().unwrap().split_once(".").unwrap().0;
+        let full_label = format!("{:8} {:40}", $geo, label);
+		let repeats = 32;
+		let ks = 256;
+        if full_label.contains($filter.unwrap_or("")) {
+            let time = b1!({
+
+				let mut p = F32;
+				let mut q = F32;
+				let mut k = ks;
+				let mut r = repeats;
+				asm!(
+					concat!(r#"
+2:
+      mov rax, r9
+      mov rcx, r10
+      mov r8, r12
+3:
+    "#, include_str!(			concat!("../x86_64/", $arch, "/", $path)), "\n sub r8, ", $u, r#"
+jnz 3b
+
+sub r11, 1
+jnz 2b
+"#),
+					inout("r9") p, inout("r10") q, inout("r12") k, inout("r11") r, out("rax") _, out("rcx") _,
+					out("r8") _,
+					out("zmm0") _, out("zmm1") _, out("zmm2") _, out("zmm3") _,
+					out("zmm4") _, out("zmm5") _, out("zmm6") _, out("zmm7") _,
+					out("zmm8") _, out("zmm9") _, out("zmm10") _, out("zmm11") _,
+					out("zmm12") _, out("zmm13") _, out("zmm14") _, out("zmm15") _,
+					out("zmm20") _, out("zmm21") _, out("zmm22") _, out("zmm23") _,
+					out("zmm24") _, out("zmm25") _, out("zmm26") _, out("zmm27") _,
+                    out("zmm28") _,  out("zmm29") _,  out("zmm30") _,  out("zmm31") _,
+				);
+            });
+
+			// We have k=1024 * 64 but some tests step twice per iteration
+			let iterations = (ks * repeats / $u);
+			// Those that step twice process twice as many elements per iteration
+			let elems_per_iteration = $n * $u;
+
+			let time_per_iteration = time / iterations  as f64;
+
+			let total_floats = elems_per_iteration * iterations;
+			let flops = total_floats as f64 / time;
+
+			let total_time_ms = time * 1e6;
+			let fmas_per_iteration = ($n as f64 / $ww as f64) * $u as f64;
+			let ticks_per_iteration = time_per_iteration / *TICK;
+            println!("{} {:3.5} {:3.0}% ({:>5.2 }/{:3 } cy) {:.2} GFLOP/s", full_label, total_time_ms, fmas_per_iteration / ticks_per_iteration * 100., ticks_per_iteration, fmas_per_iteration, flops / 1e9 );
+        }
+    };
+
+	($filter: expr, $geo: literal, $n: expr, $path: literal, $ww: expr) => {
+		kloop!($filter, $geo, $n, $path, $ww, 1, "fma")
+	};
+	($filter: expr, $geo: literal, $n: expr, $path: literal, $ww: expr, $u: expr) => {
+		kloop!($filter, $geo, $n, $path, $ww, $u, "fma")
+	};
+}
+
+unsafe fn packed_packed_1x12(f: Option<&str>) {
+    println!("-- 1x12 kernels");
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "1x12x1",
+            (16 * 1 * 12),
+            "1x12/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_1x8(f: Option<&str>) {
+    println!("-- 1x8 kernels");
+    kloop!(f, "1x8x1", (8 * 8), "8x8/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "1x8x2",
+        (8 * 8),
+        "8x8/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "1x8x1",
+            (16 * 1 * 8),
+            "8x8/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_2x6(f: Option<&str>) {
+    println!("-- 2x6 kernels");
+    kloop!(
+        f,
+        "2x6x1",
+        (16 * 6),
+        "2x6/packed_packed_loop1/original.tmpli",
+        8
+    );
+    kloop!(
+        f,
+        "2x6x2",
+        (16 * 6),
+        "2x6/packed_packed_loop1/original-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "2x6x1",
+            (16 * 2 * 6),
+            "2x6/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "2x6x2",
+            (16 * 2 * 6),
+            "2x6/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_2x5(f: Option<&str>) {
+    println!("-- 2x5 kernels");
+    kloop!(f, "2x5x1", (16 * 5), "2x5/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "2x5x2",
+        (16 * 5),
+        "2x5/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "2x5x1",
+            (32 * 5),
+            "2x5/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "2x5x2",
+            (32 * 5),
+            "2x5/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_3x4(f: Option<&str>) {
+    println!("-- 3x4 kernels");
+    kloop!(f, "3x4x1", (24 * 4), "3x4/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "3x4x2",
+        (24 * 4),
+        "3x4/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "3x4x1",
+            (16 * 3 * 4),
+            "3x4/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "3x4x2",
+            (16 * 3 * 4),
+            "3x4/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_4x3(f: Option<&str>) {
+    println!("-- 4x3 kernels");
+    kloop!(f, "4x3x1", (32 * 3), "4x3/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "4x3x2",
+        (32 * 3),
+        "4x3/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "4x3x1",
+            (16 * 4 * 3),
+            "4x3/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "4x3x2",
+            (16 * 4 * 3),
+            "4x3/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_5x2(f: Option<&str>) {
+    println!("-- 5x2 kernels");
+    kloop!(f, "5x2x1", (40 * 2), "5x2/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "5x2x1",
+        (40 * 2),
+        "5x2/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "5x2x1",
+            (16 * 5 * 2),
+            "5x2/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "5x2x2",
+            (16 * 5 * 2),
+            "5x2/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_6x2(f: Option<&str>) {
+    println!("-- 6x2 kernels");
+    kloop!(f, "6x2x1", (48 * 2), "6x2/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "6x2x2",
+        (48 * 2),
+        "6x2/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "6x2x1",
+            (16 * 6 * 2),
+            "6x2/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "6x2x2",
+            (16 * 6 * 2),
+            "6x2/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_8x2(f: Option<&str>) {
+    println!("-- 8x2 kernels");
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "8x2x1",
+            (16 * 8 * 2),
+            "8x2/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_8x1(f: Option<&str>) {
+    println!("-- 8x1 kernels");
+    kloop!(f, "8x1x1", (64 * 1), "8x1/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "8x1x2",
+        (64 * 1),
+        "8x1/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "8x1x1",
+            (16 * 8 * 1),
+            "8x1/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "8x1x2",
+            (16 * 8 * 1),
+            "8x1/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_6x1(f: Option<&str>) {
+    println!("-- 6x1 kernels");
+    kloop!(f, "6x1x1", (48 * 1), "6x1/packed_packed_loop1/avx.tmpli", 8);
+    kloop!(
+        f,
+        "6x1x2",
+        (48 * 1),
+        "6x1/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "6x1x1",
+            (16 * 6 * 1),
+            "6x1/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "6x1x2",
+            (16 * 6 * 1),
+            "6x1/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_7x1(f: Option<&str>) {
+    println!("-- 7x1 kernels");
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "7x1x1",
+            (16 * 7 * 1),
+            "7x1/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "7x1x2",
+            (16 * 7 * 1),
+            "7x1/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_1x1(f: Option<&str>) {
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "1x1x1",
+            (16 * 1 * 1),
+            "1x1/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "1x1x2",
+            (16 * 1 * 1),
+            "1x1/packed_packed_loop1/unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "1x1x4",
+            (16 * 1 * 1),
+            "1x1/packed_packed_loop1/unroll-4.tmpli",
+            16,
+            4,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "1x1x8",
+            (16 * 1 * 1),
+            "1x1/packed_packed_loop1/unroll-8.tmpli",
+            16,
+            8,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "1x1x16",
+            (16 * 1 * 1),
+            "1x1/packed_packed_loop1/unroll-16.tmpli",
+            16,
+            16,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+unsafe fn packed_packed_10x1(f: Option<&str>) {
+    println!("-- 10x1 kernels");
+    kloop!(
+        f,
+        "10x1x1",
+        (80 * 1),
+        "10x1/packed_packed_loop1/avx.tmpli",
+        8
+    );
+    kloop!(
+        f,
+        "10x1x2",
+        (80 * 1),
+        "10x1/packed_packed_loop1/avx-unroll.tmpli",
+        8,
+        2
+    );
+    if std::is_x86_feature_detected!("avx512f") {
+        kloop!(
+            f,
+            "10x1x1",
+            (16 * 10 * 1),
+            "10x1/packed_packed_loop1/avx-512.tmpli",
+            16,
+            1,
+            "avx512"
+        );
+        kloop!(
+            f,
+            "10x1x2",
+            (16 * 10 * 1),
+            "10x1/packed_packed_loop1/avx-512-unroll.tmpli",
+            16,
+            2,
+            "avx512"
+        );
+    }
+    println!();
+}
+
+fn main() {
+    let filter = std::env::args().skip(1).find(|a| a != "--bench");
+    unsafe {
+        packed_packed_1x1(filter.as_deref());
+        packed_packed_1x12(filter.as_deref());
+        packed_packed_1x8(filter.as_deref());
+        packed_packed_2x6(filter.as_deref());
+        packed_packed_2x5(filter.as_deref());
+        packed_packed_3x4(filter.as_deref());
+        packed_packed_4x3(filter.as_deref());
+        packed_packed_5x2(filter.as_deref());
+        packed_packed_6x2(filter.as_deref());
+        packed_packed_8x2(filter.as_deref());
+        packed_packed_6x1(filter.as_deref());
+        packed_packed_7x1(filter.as_deref());
+        packed_packed_8x1(filter.as_deref());
+        packed_packed_10x1(filter.as_deref());
+    }
+}