brenden/fluxer-desktop-patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add native self-hosted instance connection to fluxer_desktop

Browse Source 
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 +
tools/ci) with a "Connect to a Different Server" menu item and popout
that lets the desktop app switch to any self-hosted Fluxer instance,
plus fixes for well-known discovery on single-domain self-hosted
deployments and a false-positive ERR_ABORTED on same-origin client
redirects during the switch. Defaults to chat.fluxr.chat and uses an
isolated userData directory from the official build.
This commit is contained in:
brenden
2026-07-01 18:22:43 -04:00
commit 682afacd30
1763 changed files with 613720 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fluxer_desktop/native/webrtc-sender/vendor/tract-linalg-0.23.1/arm64/sve/sve_mmm_f16.c
Vendored
+153
View File
@@ -0,0 +1,153 @@
// SVE f16 GEMM kernel for tract's MMM framework (the mmm_f16 slot).
//
// Tile MR=8 x NR=8 with native f16 accumulation, the f16 sibling of
// sve_mmm_f32.c. The hot AddMatMul is the same vector-length-agnostic
// broadcast-A rank-1 update, but over f16 lanes: the NR columns are walked in
// svcnth() chunks with whilelt predication and folded with svmla_n_f16 (native
// f16 fused multiply-add, as the NEON arm64fp16 kernels do), so one binary is
// correct and full-width at any SVE VL 128..2048-bit.
//
// Gated on FEAT_SVE2 AND FEAT_FP16 (Rust side). Built with +fp16. Consumes
// tract's native f16 K-major packing. Fuse ops act on the MRxNR tile in memory
// (scalar; not the hot path); as with the f32 kernel, LeakyRelu and the i32
// quantization ops are excluded by CAN_FUSE. Returns 0 on success, 1 otherwise.
#include <arm_sve.h>
#include <stdint.h>
#include <string.h>
#define MR 8
#define NR 8
enum {
DONE = 0, CLEAR, LOAD_TILE,
SCALAR_MIN, SCALAR_MAX, SCALAR_ADD, SCALAR_MUL, SCALAR_SUB, SCALAR_SUBF,
LEAKY_RELU,
PER_ROW_MIN, PER_ROW_MAX, PER_ROW_ADD, PER_ROW_MUL, PER_ROW_SUB, PER_ROW_SUBF,
PER_COL_MIN, PER_COL_MAX, PER_COL_ADD, PER_COL_MUL, PER_COL_SUB, PER_COL_SUBF,
Q_SCALE, Q_SHR, Q_SHL,
ADD_UNICAST, ADD_ROW_COL_PRODUCTS, STORE, ADD_MAT_MUL
};
typedef struct {
uint64_t disc;
uint64_t f0, f1, f2, f3;
} spec_t;
static inline __fp16 f16_of(uint64_t bits) {
__fp16 f;
uint16_t lo = (uint16_t)bits;
memcpy(&f, &lo, 2);
return f;
}
// AddMatMul: ab[m][n] += sum_k pa[k*MR+m] * pb[k*NR+n]. VLA over NR (f16 lanes).
static void add_mat_mul(__fp16 ab[MR][NR], const __fp16 *pa, const __fp16 *pb, long k) {
for (long n0 = 0; n0 < NR; n0 += svcnth()) {
svbool_t pg = svwhilelt_b16((uint64_t)n0, (uint64_t)NR);
svfloat16_t a0 = svld1_f16(pg, &ab[0][n0]), a1 = svld1_f16(pg, &ab[1][n0]);
svfloat16_t a2 = svld1_f16(pg, &ab[2][n0]), a3 = svld1_f16(pg, &ab[3][n0]);
svfloat16_t a4 = svld1_f16(pg, &ab[4][n0]), a5 = svld1_f16(pg, &ab[5][n0]);
svfloat16_t a6 = svld1_f16(pg, &ab[6][n0]), a7 = svld1_f16(pg, &ab[7][n0]);
for (long kk = 0; kk < k; kk++) {
svfloat16_t b = svld1_f16(pg, &pb[kk * NR + n0]);
const __fp16 *arow = &pa[kk * MR];
a0 = svmla_n_f16_x(pg, a0, b, arow[0]);
a1 = svmla_n_f16_x(pg, a1, b, arow[1]);
a2 = svmla_n_f16_x(pg, a2, b, arow[2]);
a3 = svmla_n_f16_x(pg, a3, b, arow[3]);
a4 = svmla_n_f16_x(pg, a4, b, arow[4]);
a5 = svmla_n_f16_x(pg, a5, b, arow[5]);
a6 = svmla_n_f16_x(pg, a6, b, arow[6]);
a7 = svmla_n_f16_x(pg, a7, b, arow[7]);
}
svst1_f16(pg, &ab[0][n0], a0); svst1_f16(pg, &ab[1][n0], a1);
svst1_f16(pg, &ab[2][n0], a2); svst1_f16(pg, &ab[3][n0], a3);
svst1_f16(pg, &ab[4][n0], a4); svst1_f16(pg, &ab[5][n0], a5);
svst1_f16(pg, &ab[6][n0], a6); svst1_f16(pg, &ab[7][n0], a7);
}
}
// Store the MRxNR f16 tile with arbitrary row/col byte strides.
static void store_tile(__fp16 ab[MR][NR], const spec_t *s) {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, cstride = (long)s->f2, isz = (long)s->f3;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) {
uint8_t *p = ptr + i * rstride + j * cstride;
if (isz == 2)
*(__fp16 *)p = ab[i][j];
else if (isz == 4)
*(float *)p = (float)ab[i][j];
else
memcpy(p, &ab[i][j], isz);
}
}
intptr_t sve_mmm_f16_kernel(const spec_t *ops) {
__fp16 ab[MR][NR];
memset(ab, 0, sizeof(ab));
for (const spec_t *s = ops;; s++) {
switch (s->disc) {
case DONE:
return 0;
case CLEAR:
memset(ab, 0, sizeof(ab));
break;
case ADD_MAT_MUL: {
long k = (long)s->f0;
add_mat_mul(ab, (const __fp16 *)s->f1, (const __fp16 *)s->f2, k);
break;
}
case STORE:
store_tile(ab, s);
break;
case LOAD_TILE: {
const __fp16 *src = (const __fp16 *)s->f1;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] = src[i * NR + j];
break;
}
case ADD_UNICAST: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, cstride = (long)s->f2, isz = (long)s->f3;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) {
const uint8_t *p = ptr + i * rstride + j * cstride;
if (isz == 2)
ab[i][j] += *(const __fp16 *)p;
else
ab[i][j] += (__fp16) * (const float *)p;
}
break;
}
case ADD_ROW_COL_PRODUCTS: {
const __fp16 *rows = (const __fp16 *)s->f0;
const __fp16 *cols = (const __fp16 *)s->f1;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] += rows[i] * cols[j];
break;
}
case SCALAR_MIN: { __fp16 v=f16_of(s->f0); for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<v?ab[i][j]:v; break; }
case SCALAR_MAX: { __fp16 v=f16_of(s->f0); for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>v?ab[i][j]:v; break; }
case SCALAR_ADD: { __fp16 v=f16_of(s->f0); for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]+=v; break; }
case SCALAR_MUL: { __fp16 v=f16_of(s->f0); for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]*=v; break; }
case SCALAR_SUB: { __fp16 v=f16_of(s->f0); for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=v-ab[i][j]; break; }
case SCALAR_SUBF:{ __fp16 v=f16_of(s->f0); for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-v; break; }
case PER_ROW_MIN: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<m[i]?ab[i][j]:m[i]; break; }
case PER_ROW_MAX: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>m[i]?ab[i][j]:m[i]; break; }
case PER_ROW_ADD: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]+=m[i]; break; }
case PER_ROW_MUL: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]*=m[i]; break; }
case PER_ROW_SUB: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=m[i]-ab[i][j]; break; }
case PER_ROW_SUBF:{ const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-m[i]; break; }
case PER_COL_MIN: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<m[j]?ab[i][j]:m[j]; break; }
case PER_COL_MAX: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>m[j]?ab[i][j]:m[j]; break; }
case PER_COL_ADD: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]+=m[j]; break; }
case PER_COL_MUL: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]*=m[j]; break; }
case PER_COL_SUB: { const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=m[j]-ab[i][j]; break; }
case PER_COL_SUBF:{ const __fp16*m=(const __fp16*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-m[j]; break; }
default:
return 1;
}
}
}
fluxer_desktop/native/webrtc-sender/vendor/tract-linalg-0.23.1/arm64/sve/sve_mmm_f32.c
Vendored
+159
View File
@@ -0,0 +1,159 @@
// SVE f32 GEMM kernel for tract's MMM framework.
//
// Tile MR=8 x NR=8. The hot AddMatMul uses the vector-length-agnostic
// broadcast-A rank-1 update: the NR columns are walked in svcntw() chunks with
// whilelt predication, so the SAME code runs correctly (and uses the full
// vector) at any SVE vector length — 128 to 2048-bit. The MR=8 accumulators are
// held in SVE registers across the K loop. Fuse ops operate on an MRxNR tile in
// memory (scalar C — they are not the hot path), mirroring the generic kernel.
//
// ABI: the kernel walks a *const FusedKerSpec<f32> array (40 bytes / entry,
// discriminant u64 at offset 0, fields at 8/16/24/32) until Done, exactly like
// the asm dispatcher. f32 GEMM consumes tract's native K-major packing
// (pa[k*MR+m], pb[k*NR+n]) so no custom packing format is required.
//
// Returns 0 on success, 1 if asked to do an unsupported fused op.
#include <arm_sve.h>
#include <stdint.h>
#include <string.h>
#define MR 8
#define NR 8
// FusedKerSpec discriminants (must match frame/mmm/fuse.rs enum order).
enum {
DONE = 0, CLEAR, LOAD_TILE,
SCALAR_MIN, SCALAR_MAX, SCALAR_ADD, SCALAR_MUL, SCALAR_SUB, SCALAR_SUBF,
LEAKY_RELU,
PER_ROW_MIN, PER_ROW_MAX, PER_ROW_ADD, PER_ROW_MUL, PER_ROW_SUB, PER_ROW_SUBF,
PER_COL_MIN, PER_COL_MAX, PER_COL_ADD, PER_COL_MUL, PER_COL_SUB, PER_COL_SUBF,
Q_SCALE, Q_SHR, Q_SHL,
ADD_UNICAST, ADD_ROW_COL_PRODUCTS, STORE, ADD_MAT_MUL
};
typedef struct {
uint64_t disc;
uint64_t f0, f1, f2, f3; // fields at byte offsets 8, 16, 24, 32
} spec_t;
// AddMatMul: ab[m][n] += sum_k pa[k*MR+m] * pb[k*NR+n]. VLA over NR.
static void add_mat_mul(float ab[MR][NR], const float *pa, const float *pb, long k) {
for (long n0 = 0; n0 < NR; n0 += svcntw()) {
svbool_t pg = svwhilelt_b32((uint64_t)n0, (uint64_t)NR);
svfloat32_t a0 = svld1_f32(pg, &ab[0][n0]), a1 = svld1_f32(pg, &ab[1][n0]);
svfloat32_t a2 = svld1_f32(pg, &ab[2][n0]), a3 = svld1_f32(pg, &ab[3][n0]);
svfloat32_t a4 = svld1_f32(pg, &ab[4][n0]), a5 = svld1_f32(pg, &ab[5][n0]);
svfloat32_t a6 = svld1_f32(pg, &ab[6][n0]), a7 = svld1_f32(pg, &ab[7][n0]);
for (long kk = 0; kk < k; kk++) {
svfloat32_t b = svld1_f32(pg, &pb[kk * NR + n0]);
const float *arow = &pa[kk * MR];
a0 = svmla_n_f32_x(pg, a0, b, arow[0]);
a1 = svmla_n_f32_x(pg, a1, b, arow[1]);
a2 = svmla_n_f32_x(pg, a2, b, arow[2]);
a3 = svmla_n_f32_x(pg, a3, b, arow[3]);
a4 = svmla_n_f32_x(pg, a4, b, arow[4]);
a5 = svmla_n_f32_x(pg, a5, b, arow[5]);
a6 = svmla_n_f32_x(pg, a6, b, arow[6]);
a7 = svmla_n_f32_x(pg, a7, b, arow[7]);
}
svst1_f32(pg, &ab[0][n0], a0); svst1_f32(pg, &ab[1][n0], a1);
svst1_f32(pg, &ab[2][n0], a2); svst1_f32(pg, &ab[3][n0], a3);
svst1_f32(pg, &ab[4][n0], a4); svst1_f32(pg, &ab[5][n0], a5);
svst1_f32(pg, &ab[6][n0], a6); svst1_f32(pg, &ab[7][n0], a7);
}
}
static inline float f32_of(uint64_t bits) {
float f;
uint32_t lo = (uint32_t)bits;
memcpy(&f, &lo, 4);
return f;
}
// Store the MRxNR tile to memory with arbitrary row/col byte strides.
static void store_tile(float ab[MR][NR], const spec_t *s) {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, cstride = (long)s->f2, isz = (long)s->f3;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) {
uint8_t *p = ptr + i * rstride + j * cstride;
if (isz == 4)
*(float *)p = ab[i][j];
else
memcpy(p, &ab[i][j], isz);
}
}
// Returns isize (64-bit) to match tract's kernel ABI — NOT int (would leave the
// upper 32 bits of x0 undefined).
intptr_t sve_mmm_f32_kernel(const spec_t *ops) {
float ab[MR][NR];
memset(ab, 0, sizeof(ab));
for (const spec_t *s = ops;; s++) {
switch (s->disc) {
case DONE:
return 0;
case CLEAR:
memset(ab, 0, sizeof(ab));
break;
case ADD_MAT_MUL: {
long k = (long)s->f0;
const float *pa = (const float *)s->f1;
const float *pb = (const float *)s->f2;
add_mat_mul(ab, pa, pb, k);
break;
}
case STORE:
store_tile(ab, s);
break;
case LOAD_TILE: {
// LoadTile(col_major_ptr, row_major_ptr); use the row-major one.
const float *src = (const float *)s->f1;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] = src[i * NR + j];
break;
}
case ADD_UNICAST: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, cstride = (long)s->f2;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++)
ab[i][j] += *(const float *)(ptr + i * rstride + j * cstride);
break;
}
case ADD_ROW_COL_PRODUCTS: {
const float *rows = (const float *)s->f0;
const float *cols = (const float *)s->f1;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] += rows[i] * cols[j];
break;
}
// ---- scalar fuse ops ----
case SCALAR_MIN: { float v = f32_of(s->f0); for (long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<v?ab[i][j]:v; break; }
case SCALAR_MAX: { float v = f32_of(s->f0); for (long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>v?ab[i][j]:v; break; }
case SCALAR_ADD: { float v = f32_of(s->f0); for (long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]+=v; break; }
case SCALAR_MUL: { float v = f32_of(s->f0); for (long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]*=v; break; }
case SCALAR_SUB: { float v = f32_of(s->f0); for (long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=v-ab[i][j]; break; }
case SCALAR_SUBF:{ float v = f32_of(s->f0); for (long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-v; break; }
// ---- per-row fuse ops (one value per m-row) ----
case PER_ROW_MIN: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<m[i]?ab[i][j]:m[i]; break; }
case PER_ROW_MAX: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>m[i]?ab[i][j]:m[i]; break; }
case PER_ROW_ADD: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]+=m[i]; break; }
case PER_ROW_MUL: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]*=m[i]; break; }
case PER_ROW_SUB: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=m[i]-ab[i][j]; break; }
case PER_ROW_SUBF:{ const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-m[i]; break; }
// ---- per-col fuse ops (one value per n-col) ----
case PER_COL_MIN: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<m[j]?ab[i][j]:m[j]; break; }
case PER_COL_MAX: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>m[j]?ab[i][j]:m[j]; break; }
case PER_COL_ADD: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]+=m[j]; break; }
case PER_COL_MUL: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]*=m[j]; break; }
case PER_COL_SUB: { const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=m[j]-ab[i][j]; break; }
case PER_COL_SUBF:{ const float*m=(const float*)s->f0; for(long i=0;i<MR;i++) for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-m[j]; break; }
default:
// LeakyRelu / QScale / RoundingShiftRight / ShiftLeft: excluded
// by CAN_FUSE for f32 — should never arrive. Anything else: error.
return 1;
}
}
}
fluxer_desktop/native/webrtc-sender/vendor/tract-linalg-0.23.1/arm64/sve/sve_mmm_i32.c
Vendored
+261
View File
@@ -0,0 +1,261 @@
// SVE int8 -> int32 GEMM kernel for tract's MMM framework (the qmmm_i32 slot).
//
// Tile MR=8 x NR=8, i32 accumulator. The hot AddMatMul is the vector-length-
// agnostic *widening* broadcast-A rank-1 update: per K-step it loads NR signed
// bytes of a B row and sign-extends them to i32 with `svld1sb_s32`, then folds
// MR `svmla_n_s32` updates with the (sign-extended) scalar from the A column.
// The NR columns are walked in svcntw() chunks with whilelt predication, so the
// SAME code is correct and full-width at any SVE vector length (128..2048-bit).
//
// Why widening MLA and not SDOT: SDOT reduces 4 K-contiguous i8 per i32 lane, so
// it needs A and B packed K-contiguous within a lane. tract's PackedFormat is
// K-major (mn-inner: for each k, r contiguous mn values), which is exactly what
// the per-k widening update consumes — and what arm64simd's i32 kernel uses via
// NEON SMLAL. A SDOT/SMMLA path would need a custom interleaved packer; that is
// a separate (max-throughput) kernel, not this one.
//
// int8 inputs arrive via tract's native i8i8 packing (AddMatMul packing == 1).
// The default i32i32 packing (packing == 0) is also handled (scalar) so the
// generic auto-test surface (mmm_packed_packed_tests i32i32:0) passes.
//
// ABI: identical 40-byte FusedKerSpec<i32> walk as the f32 kernel (discriminant
// u64 at offset 0, fields at 8/16/24/32). Fuse ops act on the MRxNR i32 tile in
// memory (scalar C — not the hot path), including the quantization ops
// q_scale / q_shr (rounding) / q_shl, ported bit-exact from
// linalg/src/generic/rounding.rs.
//
// Returns 0 on success, 1 if asked to do an unsupported fused op / packing.
#include <arm_sve.h>
#include <stdint.h>
#include <string.h>
#define MR 8
#define NR 8
// FusedKerSpec discriminants (must match frame/mmm/fuse.rs enum order).
enum {
DONE = 0, CLEAR, LOAD_TILE,
SCALAR_MIN, SCALAR_MAX, SCALAR_ADD, SCALAR_MUL, SCALAR_SUB, SCALAR_SUBF,
LEAKY_RELU,
PER_ROW_MIN, PER_ROW_MAX, PER_ROW_ADD, PER_ROW_MUL, PER_ROW_SUB, PER_ROW_SUBF,
PER_COL_MIN, PER_COL_MAX, PER_COL_ADD, PER_COL_MUL, PER_COL_SUB, PER_COL_SUBF,
Q_SCALE, Q_SHR, Q_SHL,
ADD_UNICAST, ADD_ROW_COL_PRODUCTS, STORE, ADD_MAT_MUL
};
// RoundingPolicy is #[repr(usize)] in fuse.rs: Native=0, Zero=1, Away=2,
// MinusInf=3, PlusInf=4, Even=5, Odd=6.
enum { RP_NATIVE = 0, RP_ZERO, RP_AWAY, RP_MINUSINF, RP_PLUSINF, RP_EVEN, RP_ODD };
typedef struct {
uint64_t disc;
uint64_t f0, f1, f2, f3; // fields at byte offsets 8, 16, 24, 32
} spec_t;
// AddMatMul, i8 x i8 -> i32 (packing 1): ab[m][n] += sum_k pa[k*MR+m]*pb[k*NR+n].
// VLA widening rank-1 update over NR.
static void add_mat_mul_i8(int32_t ab[MR][NR], const int8_t *pa, const int8_t *pb, long k) {
for (long n0 = 0; n0 < NR; n0 += svcntw()) {
svbool_t pg = svwhilelt_b32((uint64_t)n0, (uint64_t)NR);
svint32_t a0 = svld1_s32(pg, &ab[0][n0]), a1 = svld1_s32(pg, &ab[1][n0]);
svint32_t a2 = svld1_s32(pg, &ab[2][n0]), a3 = svld1_s32(pg, &ab[3][n0]);
svint32_t a4 = svld1_s32(pg, &ab[4][n0]), a5 = svld1_s32(pg, &ab[5][n0]);
svint32_t a6 = svld1_s32(pg, &ab[6][n0]), a7 = svld1_s32(pg, &ab[7][n0]);
for (long kk = 0; kk < k; kk++) {
// Load NR int8 of B row kk, sign-extending each lane to i32.
svint32_t b = svld1sb_s32(pg, &pb[kk * NR + n0]);
const int8_t *arow = &pa[kk * MR];
a0 = svmla_n_s32_x(pg, a0, b, (int32_t)arow[0]);
a1 = svmla_n_s32_x(pg, a1, b, (int32_t)arow[1]);
a2 = svmla_n_s32_x(pg, a2, b, (int32_t)arow[2]);
a3 = svmla_n_s32_x(pg, a3, b, (int32_t)arow[3]);
a4 = svmla_n_s32_x(pg, a4, b, (int32_t)arow[4]);
a5 = svmla_n_s32_x(pg, a5, b, (int32_t)arow[5]);
a6 = svmla_n_s32_x(pg, a6, b, (int32_t)arow[6]);
a7 = svmla_n_s32_x(pg, a7, b, (int32_t)arow[7]);
}
svst1_s32(pg, &ab[0][n0], a0); svst1_s32(pg, &ab[1][n0], a1);
svst1_s32(pg, &ab[2][n0], a2); svst1_s32(pg, &ab[3][n0], a3);
svst1_s32(pg, &ab[4][n0], a4); svst1_s32(pg, &ab[5][n0], a5);
svst1_s32(pg, &ab[6][n0], a6); svst1_s32(pg, &ab[7][n0], a7);
}
}
// AddMatMul, i32 x i32 -> i32 (packing 0, default): only used by the auto-test
// surface, never in production (quantized matmul uses the i8i8 packing). Scalar.
static void add_mat_mul_i32(int32_t ab[MR][NR], const int32_t *pa, const int32_t *pb, long k) {
for (long kk = 0; kk < k; kk++) {
const int32_t *arow = &pa[kk * MR], *brow = &pb[kk * NR];
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] += arow[i] * brow[j];
}
}
// ---- Quantization helpers, ported bit-exact from generic/rounding.rs ----
// i32::q_shr(shift, rp): rounding arithmetic shift right.
static int32_t q_shr_i32(int32_t v, long shift, int rp) {
int32_t half = (int32_t)1 << (shift - 1);
int32_t a = v < 0 ? -v : v; // abs (test inputs are small; matches Rust .abs())
int32_t nudge;
switch (rp) {
case RP_ZERO: nudge = -1; break;
case RP_MINUSINF: nudge = -(int32_t)(v >= 0); break;
case RP_PLUSINF: nudge = -(int32_t)(v <= 0); break;
case RP_AWAY: nudge = 0; break;
case RP_EVEN: nudge = ((a >> shift) & 0x1) - 1; break;
case RP_ODD: nudge = -((a >> shift) & 0x1); break;
default: nudge = 0; break; // Native: unreachable for q ops
}
int32_t sign = (v > 0) - (v < 0); // signum: -1 / 0 / 1
return sign * ((a + half + nudge) >> shift);
}
// i32::q_scale(Scaler{mult, shift, policy}) with mult always present (the QScale
// fused op carries an explicit multiplier). Mirrors `Mul<i32> for Scaler`.
static int32_t q_scale_i32(int32_t v, long shift_in, int policy, int32_t mult) {
int64_t val = (int64_t)mult * (int64_t)v;
long shift = shift_in + 31;
if (shift > 0) {
int64_t half = (int64_t)1 << (shift - 1);
int64_t a = val < 0 ? -val : val;
int64_t nudge;
switch (policy) {
case RP_ZERO: nudge = -1; break;
case RP_MINUSINF: nudge = -(int64_t)(val >= 0); break;
case RP_PLUSINF: nudge = -(int64_t)(val <= 0); break;
case RP_AWAY: nudge = 0; break;
case RP_EVEN: nudge = ((a >> shift) & 0x1) - 1; break;
case RP_ODD: nudge = -((a >> shift) & 0x1); break;
default: nudge = 0; break;
}
int64_t sign = (val > 0) - (val < 0);
return (int32_t)(sign * ((a + half + nudge) >> shift));
} else {
return (int32_t)(val << (-shift));
}
}
// Store the MRxNR i32 tile to memory with arbitrary row/col byte strides,
// truncating to the destination item size (matches generic store_t semantics
// for the tested widths 1 and 4).
static void store_tile(int32_t ab[MR][NR], const spec_t *s) {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, cstride = (long)s->f2, isz = (long)s->f3;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) {
uint8_t *p = ptr + i * rstride + j * cstride;
int32_t v = ab[i][j];
switch (isz) {
case 1: *(uint8_t *)p = (uint8_t)v; break;
case 2: *(uint16_t *)p = (uint16_t)v; break;
case 4: *(int32_t *)p = v; break;
case 8: { int64_t w = v; memcpy(p, &w, 8); break; }
default: memcpy(p, &v, isz < 4 ? (size_t)isz : 4); break;
}
}
}
// Returns isize (64-bit) to match tract's kernel ABI.
intptr_t sve_mmm_i32_kernel(const spec_t *ops) {
int32_t ab[MR][NR];
memset(ab, 0, sizeof(ab));
for (const spec_t *s = ops;; s++) {
switch (s->disc) {
case DONE:
return 0;
case CLEAR:
memset(ab, 0, sizeof(ab));
break;
case ADD_MAT_MUL: {
long k = (long)s->f0;
long packing = (long)s->f3;
if (packing == 1) {
add_mat_mul_i8(ab, (const int8_t *)s->f1, (const int8_t *)s->f2, k);
} else if (packing == 0) {
add_mat_mul_i32(ab, (const int32_t *)s->f1, (const int32_t *)s->f2, k);
} else {
return 1;
}
break;
}
case STORE:
store_tile(ab, s);
break;
case LOAD_TILE: {
// LoadTile(col_major_ptr, row_major_ptr); use the row-major one.
const int32_t *src = (const int32_t *)s->f1;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] = src[i * NR + j];
break;
}
case ADD_UNICAST: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, cstride = (long)s->f2, isz = (long)s->f3;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) {
const uint8_t *p = ptr + i * rstride + j * cstride;
if (isz == 1)
ab[i][j] += *(const int8_t *)p; // sign-extend
else
ab[i][j] += *(const int32_t *)p;
}
break;
}
case ADD_ROW_COL_PRODUCTS: {
const int32_t *rows = (const int32_t *)s->f0;
const int32_t *cols = (const int32_t *)s->f1;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] += rows[i] * cols[j];
break;
}
// ---- quantization fuse ops ----
case Q_SCALE: {
long shift = (long)s->f0;
int policy = (int)s->f1;
int32_t mult = (int32_t)s->f2;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] = q_scale_i32(ab[i][j], shift, policy, mult);
break;
}
case Q_SHR: {
long shift = (long)s->f0;
int policy = (int)s->f1;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] = q_shr_i32(ab[i][j], shift, policy);
break;
}
case Q_SHL: {
long shift = (long)s->f0;
for (long i = 0; i < MR; i++)
for (long j = 0; j < NR; j++) ab[i][j] = ab[i][j] << shift;
break;
}
// ---- scalar fuse ops (value is an i32 in the low 32 bits of f0) ----
case SCALAR_MIN: { int32_t v=(int32_t)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<v?ab[i][j]:v; break; }
case SCALAR_MAX: { int32_t v=(int32_t)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>v?ab[i][j]:v; break; }
case SCALAR_ADD: { int32_t v=(int32_t)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]+=v; break; }
case SCALAR_MUL: { int32_t v=(int32_t)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]*=v; break; }
case SCALAR_SUB: { int32_t v=(int32_t)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=v-ab[i][j]; break; }
case SCALAR_SUBF:{ int32_t v=(int32_t)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-v; break; }
// ---- per-row fuse ops (one i32 per m-row) ----
case PER_ROW_MIN: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<m[i]?ab[i][j]:m[i]; break; }
case PER_ROW_MAX: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>m[i]?ab[i][j]:m[i]; break; }
case PER_ROW_ADD: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]+=m[i]; break; }
case PER_ROW_MUL: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]*=m[i]; break; }
case PER_ROW_SUB: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=m[i]-ab[i][j]; break; }
case PER_ROW_SUBF:{ const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-m[i]; break; }
// ---- per-col fuse ops (one i32 per n-col) ----
case PER_COL_MIN: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]<m[j]?ab[i][j]:m[j]; break; }
case PER_COL_MAX: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]>m[j]?ab[i][j]:m[j]; break; }
case PER_COL_ADD: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]+=m[j]; break; }
case PER_COL_MUL: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]*=m[j]; break; }
case PER_COL_SUB: { const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=m[j]-ab[i][j]; break; }
case PER_COL_SUBF:{ const int32_t*m=(const int32_t*)s->f0; for(long i=0;i<MR;i++)for(long j=0;j<NR;j++) ab[i][j]=ab[i][j]-m[j]; break; }
default:
// LeakyRelu excluded by CAN_FUSE_I32; anything else is an error.
return 1;
}
}
}
fluxer_desktop/native/webrtc-sender/vendor/tract-linalg-0.23.1/arm64/sve/sve_mmm_i32_64x1.c
Vendored
+210
View File
@@ -0,0 +1,210 @@
// SVE int8 -> int32 GEMV kernel for tract's MMM framework (the qmmv_i32 slot,
// dispatched when N == 1: matrix x int8 column vector).
//
// Tile MR=64 x NR=1, i32 accumulator. The hot AddMatMul is the vector-length-
// agnostic widening update vectorized over M: per K-step it loads MR signed
// bytes of the A-panel column, sign-extends to i32 (svld1sb_s32), and folds a
// single svmla_n_s32 with the (sign-extended) scalar B[k]. The MR rows are
// walked in svcntw() chunks with whilelt predication, so the SAME code is
// correct and full-width at any SVE vector length (128..2048-bit).
//
// Same rationale as the 8x8 kernel: widening MLA (not SDOT) consumes tract's
// native K-major i8i8 packing directly. int8 inputs arrive via that packing
// (AddMatMul packing == 1); the default i32i32 packing (packing == 0) is handled
// scalar for the auto-test surface.
//
// ABI: identical 40-byte FusedKerSpec<i32> walk. At NR=1, per_col / scalar fuse
// ops degenerate to a single broadcast value and per_row is element-wise over
// the MR outputs. Quantization ops q_scale / q_shr / q_shl are ported bit-exact
// from linalg/src/generic/rounding.rs. Returns 0 on success, 1 on an
// unsupported fused op / packing.
#include <arm_sve.h>
#include <stdint.h>
#include <string.h>
#define MR 64
#define NR 1
enum {
DONE = 0, CLEAR, LOAD_TILE,
SCALAR_MIN, SCALAR_MAX, SCALAR_ADD, SCALAR_MUL, SCALAR_SUB, SCALAR_SUBF,
LEAKY_RELU,
PER_ROW_MIN, PER_ROW_MAX, PER_ROW_ADD, PER_ROW_MUL, PER_ROW_SUB, PER_ROW_SUBF,
PER_COL_MIN, PER_COL_MAX, PER_COL_ADD, PER_COL_MUL, PER_COL_SUB, PER_COL_SUBF,
Q_SCALE, Q_SHR, Q_SHL,
ADD_UNICAST, ADD_ROW_COL_PRODUCTS, STORE, ADD_MAT_MUL
};
enum { RP_NATIVE = 0, RP_ZERO, RP_AWAY, RP_MINUSINF, RP_PLUSINF, RP_EVEN, RP_ODD };
typedef struct {
uint64_t disc;
uint64_t f0, f1, f2, f3;
} spec_t;
// AddMatMul, i8 x i8 -> i32 (packing 1): ab[m] += sum_k pa[k*MR+m]*pb[k].
// VLA widening update over MR.
static void add_mat_mul_i8(int32_t ab[MR], const int8_t *pa, const int8_t *pb, long k) {
for (long m0 = 0; m0 < MR; m0 += svcntw()) {
svbool_t pg = svwhilelt_b32((uint64_t)m0, (uint64_t)MR);
svint32_t acc = svld1_s32(pg, &ab[m0]);
for (long kk = 0; kk < k; kk++) {
svint32_t a = svld1sb_s32(pg, &pa[kk * MR + m0]); // load i8 col, sign-extend
acc = svmla_n_s32_x(pg, acc, a, (int32_t)pb[kk]);
}
svst1_s32(pg, &ab[m0], acc);
}
}
// AddMatMul, i32 x i32 -> i32 (packing 0, default): auto-test surface only.
static void add_mat_mul_i32(int32_t ab[MR], const int32_t *pa, const int32_t *pb, long k) {
for (long kk = 0; kk < k; kk++) {
int32_t b = pb[kk];
const int32_t *acol = &pa[kk * MR];
for (long m = 0; m < MR; m++) ab[m] += acol[m] * b;
}
}
// ---- quantization helpers, ported bit-exact from generic/rounding.rs ----
static int32_t q_shr_i32(int32_t v, long shift, int rp) {
int32_t half = (int32_t)1 << (shift - 1);
int32_t a = v < 0 ? -v : v;
int32_t nudge;
switch (rp) {
case RP_ZERO: nudge = -1; break;
case RP_MINUSINF: nudge = -(int32_t)(v >= 0); break;
case RP_PLUSINF: nudge = -(int32_t)(v <= 0); break;
case RP_AWAY: nudge = 0; break;
case RP_EVEN: nudge = ((a >> shift) & 0x1) - 1; break;
case RP_ODD: nudge = -((a >> shift) & 0x1); break;
default: nudge = 0; break;
}
int32_t sign = (v > 0) - (v < 0);
return sign * ((a + half + nudge) >> shift);
}
static int32_t q_scale_i32(int32_t v, long shift_in, int policy, int32_t mult) {
int64_t val = (int64_t)mult * (int64_t)v;
long shift = shift_in + 31;
if (shift > 0) {
int64_t half = (int64_t)1 << (shift - 1);
int64_t a = val < 0 ? -val : val;
int64_t nudge;
switch (policy) {
case RP_ZERO: nudge = -1; break;
case RP_MINUSINF: nudge = -(int64_t)(val >= 0); break;
case RP_PLUSINF: nudge = -(int64_t)(val <= 0); break;
case RP_AWAY: nudge = 0; break;
case RP_EVEN: nudge = ((a >> shift) & 0x1) - 1; break;
case RP_ODD: nudge = -((a >> shift) & 0x1); break;
default: nudge = 0; break;
}
int64_t sign = (val > 0) - (val < 0);
return (int32_t)(sign * ((a + half + nudge) >> shift));
} else {
return (int32_t)(val << (-shift));
}
}
intptr_t sve_mmm_i32_64x1_kernel(const spec_t *ops) {
int32_t ab[MR];
memset(ab, 0, sizeof(ab));
for (const spec_t *s = ops;; s++) {
switch (s->disc) {
case DONE:
return 0;
case CLEAR:
memset(ab, 0, sizeof(ab));
break;
case ADD_MAT_MUL: {
long k = (long)s->f0, packing = (long)s->f3;
if (packing == 1)
add_mat_mul_i8(ab, (const int8_t *)s->f1, (const int8_t *)s->f2, k);
else if (packing == 0)
add_mat_mul_i32(ab, (const int32_t *)s->f1, (const int32_t *)s->f2, k);
else
return 1;
break;
}
case STORE: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, isz = (long)s->f3;
for (long m = 0; m < MR; m++) {
uint8_t *p = ptr + m * rstride;
int32_t v = ab[m];
switch (isz) {
case 1: *(uint8_t *)p = (uint8_t)v; break;
case 2: *(uint16_t *)p = (uint16_t)v; break;
case 4: *(int32_t *)p = v; break;
case 8: { int64_t w = v; memcpy(p, &w, 8); break; }
default: memcpy(p, &v, isz < 4 ? (size_t)isz : 4); break;
}
}
break;
}
case LOAD_TILE: {
const int32_t *src = (const int32_t *)s->f1; // row-major MR values
for (long m = 0; m < MR; m++) ab[m] = src[m];
break;
}
case ADD_UNICAST: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, isz = (long)s->f3;
for (long m = 0; m < MR; m++) {
const uint8_t *p = ptr + m * rstride;
if (isz == 1)
ab[m] += *(const int8_t *)p;
else
ab[m] += *(const int32_t *)p;
}
break;
}
case ADD_ROW_COL_PRODUCTS: {
const int32_t *rows = (const int32_t *)s->f0;
const int32_t *cols = (const int32_t *)s->f1;
for (long m = 0; m < MR; m++) ab[m] += rows[m] * cols[0];
break;
}
case Q_SCALE: {
long shift = (long)s->f0; int policy = (int)s->f1; int32_t mult = (int32_t)s->f2;
for (long m = 0; m < MR; m++) ab[m] = q_scale_i32(ab[m], shift, policy, mult);
break;
}
case Q_SHR: {
long shift = (long)s->f0; int policy = (int)s->f1;
for (long m = 0; m < MR; m++) ab[m] = q_shr_i32(ab[m], shift, policy);
break;
}
case Q_SHL: {
long shift = (long)s->f0;
for (long m = 0; m < MR; m++) ab[m] = ab[m] << shift;
break;
}
// scalar fuse ops (single i32 in low 32 bits of f0)
case SCALAR_MIN: { int32_t v=(int32_t)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]<v?ab[m]:v; break; }
case SCALAR_MAX: { int32_t v=(int32_t)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]>v?ab[m]:v; break; }
case SCALAR_ADD: { int32_t v=(int32_t)s->f0; for(long m=0;m<MR;m++) ab[m]+=v; break; }
case SCALAR_MUL: { int32_t v=(int32_t)s->f0; for(long m=0;m<MR;m++) ab[m]*=v; break; }
case SCALAR_SUB: { int32_t v=(int32_t)s->f0; for(long m=0;m<MR;m++) ab[m]=v-ab[m]; break; }
case SCALAR_SUBF:{ int32_t v=(int32_t)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]-v; break; }
// per-row fuse ops (one i32 per m-row)
case PER_ROW_MIN: { const int32_t*m_=(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]<m_[m]?ab[m]:m_[m]; break; }
case PER_ROW_MAX: { const int32_t*m_=(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]>m_[m]?ab[m]:m_[m]; break; }
case PER_ROW_ADD: { const int32_t*m_=(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]+=m_[m]; break; }
case PER_ROW_MUL: { const int32_t*m_=(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]*=m_[m]; break; }
case PER_ROW_SUB: { const int32_t*m_=(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=m_[m]-ab[m]; break; }
case PER_ROW_SUBF:{ const int32_t*m_=(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]-m_[m]; break; }
// per-col fuse ops degenerate to a single broadcast value at NR=1
case PER_COL_MIN: { int32_t v=*(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]<v?ab[m]:v; break; }
case PER_COL_MAX: { int32_t v=*(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]>v?ab[m]:v; break; }
case PER_COL_ADD: { int32_t v=*(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]+=v; break; }
case PER_COL_MUL: { int32_t v=*(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]*=v; break; }
case PER_COL_SUB: { int32_t v=*(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=v-ab[m]; break; }
case PER_COL_SUBF:{ int32_t v=*(const int32_t*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]-v; break; }
default:
return 1;
}
}
}
fluxer_desktop/native/webrtc-sender/vendor/tract-linalg-0.23.1/arm64/sve/sve_mmv_f16_64x1.c
Vendored
+132
View File
@@ -0,0 +1,132 @@
// SVE f16 GEMV kernel for tract's MMM framework (the mmv_f16 slot, dispatched
// when N == 1: matrix x f16 column vector).
//
// Tile MR=64 x NR=1 with native f16 accumulation, the f16 sibling of
// sve_mmv_f32_64x1.c. The hot AddMatMul is vectorized over M: per K-step it
// loads MR f16 of the A-panel column and folds a single svmla_n_f16 (native f16
// fused multiply-add) with the scalar B[k]. The MR rows are walked in svcnth()
// chunks with whilelt predication, so one binary is correct and full-width at
// any SVE VL 128..2048-bit.
//
// Gated on FEAT_SVE2 AND FEAT_FP16 (Rust side). Built with +fp16. At NR=1 the
// per_col/scalar fuse ops degenerate to a broadcast and per_row is element-wise
// over the MR outputs. LeakyRelu and the i32 quantization ops are excluded by
// CAN_FUSE. Returns 0 on success, 1 otherwise.
#include <arm_sve.h>
#include <stdint.h>
#include <string.h>
#define MR 64
#define NR 1
enum {
DONE = 0, CLEAR, LOAD_TILE,
SCALAR_MIN, SCALAR_MAX, SCALAR_ADD, SCALAR_MUL, SCALAR_SUB, SCALAR_SUBF,
LEAKY_RELU,
PER_ROW_MIN, PER_ROW_MAX, PER_ROW_ADD, PER_ROW_MUL, PER_ROW_SUB, PER_ROW_SUBF,
PER_COL_MIN, PER_COL_MAX, PER_COL_ADD, PER_COL_MUL, PER_COL_SUB, PER_COL_SUBF,
Q_SCALE, Q_SHR, Q_SHL,
ADD_UNICAST, ADD_ROW_COL_PRODUCTS, STORE, ADD_MAT_MUL
};
typedef struct {
uint64_t disc;
uint64_t f0, f1, f2, f3;
} spec_t;
static inline __fp16 f16_of(uint64_t bits) {
__fp16 f;
uint16_t lo = (uint16_t)bits;
memcpy(&f, &lo, 2);
return f;
}
// AddMatMul: ab[m] += sum_k pa[k*MR+m] * pb[k]. VLA over MR (f16 lanes).
static void add_mat_mul(__fp16 ab[MR], const __fp16 *pa, const __fp16 *pb, long k) {
for (long m0 = 0; m0 < MR; m0 += svcnth()) {
svbool_t pg = svwhilelt_b16((uint64_t)m0, (uint64_t)MR);
svfloat16_t acc = svld1_f16(pg, &ab[m0]);
for (long kk = 0; kk < k; kk++) {
svfloat16_t a = svld1_f16(pg, &pa[kk * MR + m0]);
acc = svmla_n_f16_x(pg, acc, a, pb[kk]);
}
svst1_f16(pg, &ab[m0], acc);
}
}
intptr_t sve_mmv_f16_64x1_kernel(const spec_t *ops) {
__fp16 ab[MR];
memset(ab, 0, sizeof(ab));
for (const spec_t *s = ops;; s++) {
switch (s->disc) {
case DONE:
return 0;
case CLEAR:
memset(ab, 0, sizeof(ab));
break;
case ADD_MAT_MUL: {
long k = (long)s->f0;
add_mat_mul(ab, (const __fp16 *)s->f1, (const __fp16 *)s->f2, k);
break;
}
case STORE: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, isz = (long)s->f3;
for (long m = 0; m < MR; m++) {
uint8_t *p = ptr + m * rstride;
if (isz == 2)
*(__fp16 *)p = ab[m];
else if (isz == 4)
*(float *)p = (float)ab[m];
else
memcpy(p, &ab[m], isz);
}
break;
}
case LOAD_TILE: {
const __fp16 *src = (const __fp16 *)s->f1;
for (long m = 0; m < MR; m++) ab[m] = src[m];
break;
}
case ADD_UNICAST: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, isz = (long)s->f3;
for (long m = 0; m < MR; m++) {
const uint8_t *p = ptr + m * rstride;
if (isz == 2)
ab[m] += *(const __fp16 *)p;
else
ab[m] += (__fp16) * (const float *)p;
}
break;
}
case ADD_ROW_COL_PRODUCTS: {
const __fp16 *rows = (const __fp16 *)s->f0;
const __fp16 *cols = (const __fp16 *)s->f1;
for (long m = 0; m < MR; m++) ab[m] += rows[m] * cols[0];
break;
}
case SCALAR_MIN: { __fp16 v=f16_of(s->f0); for(long m=0;m<MR;m++) ab[m]=ab[m]<v?ab[m]:v; break; }
case SCALAR_MAX: { __fp16 v=f16_of(s->f0); for(long m=0;m<MR;m++) ab[m]=ab[m]>v?ab[m]:v; break; }
case SCALAR_ADD: { __fp16 v=f16_of(s->f0); for(long m=0;m<MR;m++) ab[m]+=v; break; }
case SCALAR_MUL: { __fp16 v=f16_of(s->f0); for(long m=0;m<MR;m++) ab[m]*=v; break; }
case SCALAR_SUB: { __fp16 v=f16_of(s->f0); for(long m=0;m<MR;m++) ab[m]=v-ab[m]; break; }
case SCALAR_SUBF:{ __fp16 v=f16_of(s->f0); for(long m=0;m<MR;m++) ab[m]=ab[m]-v; break; }
case PER_ROW_MIN: { const __fp16*m_=(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]<m_[m]?ab[m]:m_[m]; break; }
case PER_ROW_MAX: { const __fp16*m_=(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]>m_[m]?ab[m]:m_[m]; break; }
case PER_ROW_ADD: { const __fp16*m_=(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]+=m_[m]; break; }
case PER_ROW_MUL: { const __fp16*m_=(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]*=m_[m]; break; }
case PER_ROW_SUB: { const __fp16*m_=(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=m_[m]-ab[m]; break; }
case PER_ROW_SUBF:{ const __fp16*m_=(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]-m_[m]; break; }
case PER_COL_MIN: { __fp16 v=*(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]<v?ab[m]:v; break; }
case PER_COL_MAX: { __fp16 v=*(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]>v?ab[m]:v; break; }
case PER_COL_ADD: { __fp16 v=*(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]+=v; break; }
case PER_COL_MUL: { __fp16 v=*(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]*=v; break; }
case PER_COL_SUB: { __fp16 v=*(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=v-ab[m]; break; }
case PER_COL_SUBF:{ __fp16 v=*(const __fp16*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]-v; break; }
default:
return 1;
}
}
}
fluxer_desktop/native/webrtc-sender/vendor/tract-linalg-0.23.1/arm64/sve/sve_mmv_f32_64x1.c
Vendored
+128
View File
@@ -0,0 +1,128 @@
// SVE f32 GEMV kernel for tract's MMM framework (the mmv_f32 slot, dispatched
// when N == 1: matrix x f32 column vector).
//
// Tile MR=64 x NR=1. The hot AddMatMul is the vector-length-agnostic update
// vectorized over M: per K-step it loads MR f32 of the A-panel column and folds
// a single svmla_n_f32 with the scalar B[k]. The MR rows are walked in svcntw()
// chunks with whilelt predication, so the SAME code is correct and full-width at
// any SVE vector length (128..2048-bit).
//
// Sibling of sve_mmm_f32.c (the 8x8 GEMM); shares the same FusedKerSpec<f32> ABI
// and fuse-op surface. At NR=1, per_col / scalar fuse ops degenerate to a single
// broadcast value and per_row is element-wise over the MR outputs. As with the
// f32 GEMM kernel, LeakyRelu and the i32 quantization ops are excluded by
// CAN_FUSE. Returns 0 on success, 1 on an unsupported fused op.
#include <arm_sve.h>
#include <stdint.h>
#include <string.h>
#define MR 64
#define NR 1
enum {
DONE = 0, CLEAR, LOAD_TILE,
SCALAR_MIN, SCALAR_MAX, SCALAR_ADD, SCALAR_MUL, SCALAR_SUB, SCALAR_SUBF,
LEAKY_RELU,
PER_ROW_MIN, PER_ROW_MAX, PER_ROW_ADD, PER_ROW_MUL, PER_ROW_SUB, PER_ROW_SUBF,
PER_COL_MIN, PER_COL_MAX, PER_COL_ADD, PER_COL_MUL, PER_COL_SUB, PER_COL_SUBF,
Q_SCALE, Q_SHR, Q_SHL,
ADD_UNICAST, ADD_ROW_COL_PRODUCTS, STORE, ADD_MAT_MUL
};
typedef struct {
uint64_t disc;
uint64_t f0, f1, f2, f3;
} spec_t;
static inline float f32_of(uint64_t bits) {
float f;
uint32_t lo = (uint32_t)bits;
memcpy(&f, &lo, 4);
return f;
}
// AddMatMul: ab[m] += sum_k pa[k*MR+m] * pb[k]. VLA over MR.
static void add_mat_mul(float ab[MR], const float *pa, const float *pb, long k) {
for (long m0 = 0; m0 < MR; m0 += svcntw()) {
svbool_t pg = svwhilelt_b32((uint64_t)m0, (uint64_t)MR);
svfloat32_t acc = svld1_f32(pg, &ab[m0]);
for (long kk = 0; kk < k; kk++) {
svfloat32_t a = svld1_f32(pg, &pa[kk * MR + m0]);
acc = svmla_n_f32_x(pg, acc, a, pb[kk]);
}
svst1_f32(pg, &ab[m0], acc);
}
}
intptr_t sve_mmv_f32_64x1_kernel(const spec_t *ops) {
float ab[MR];
memset(ab, 0, sizeof(ab));
for (const spec_t *s = ops;; s++) {
switch (s->disc) {
case DONE:
return 0;
case CLEAR:
memset(ab, 0, sizeof(ab));
break;
case ADD_MAT_MUL: {
long k = (long)s->f0;
add_mat_mul(ab, (const float *)s->f1, (const float *)s->f2, k);
break;
}
case STORE: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1, isz = (long)s->f3;
for (long m = 0; m < MR; m++) {
uint8_t *p = ptr + m * rstride;
if (isz == 4)
*(float *)p = ab[m];
else
memcpy(p, &ab[m], isz);
}
break;
}
case LOAD_TILE: {
const float *src = (const float *)s->f1; // row-major MR values
for (long m = 0; m < MR; m++) ab[m] = src[m];
break;
}
case ADD_UNICAST: {
uint8_t *ptr = (uint8_t *)s->f0;
long rstride = (long)s->f1;
for (long m = 0; m < MR; m++) ab[m] += *(const float *)(ptr + m * rstride);
break;
}
case ADD_ROW_COL_PRODUCTS: {
const float *rows = (const float *)s->f0;
const float *cols = (const float *)s->f1;
for (long m = 0; m < MR; m++) ab[m] += rows[m] * cols[0];
break;
}
// scalar fuse ops (f32 bits in low 32 bits of f0)
case SCALAR_MIN: { float v=f32_of(s->f0); for(long m=0;m<MR;m++) ab[m]=ab[m]<v?ab[m]:v; break; }
case SCALAR_MAX: { float v=f32_of(s->f0); for(long m=0;m<MR;m++) ab[m]=ab[m]>v?ab[m]:v; break; }
case SCALAR_ADD: { float v=f32_of(s->f0); for(long m=0;m<MR;m++) ab[m]+=v; break; }
case SCALAR_MUL: { float v=f32_of(s->f0); for(long m=0;m<MR;m++) ab[m]*=v; break; }
case SCALAR_SUB: { float v=f32_of(s->f0); for(long m=0;m<MR;m++) ab[m]=v-ab[m]; break; }
case SCALAR_SUBF:{ float v=f32_of(s->f0); for(long m=0;m<MR;m++) ab[m]=ab[m]-v; break; }
// per-row fuse ops (one f32 per m-row)
case PER_ROW_MIN: { const float*m_=(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]<m_[m]?ab[m]:m_[m]; break; }
case PER_ROW_MAX: { const float*m_=(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]>m_[m]?ab[m]:m_[m]; break; }
case PER_ROW_ADD: { const float*m_=(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]+=m_[m]; break; }
case PER_ROW_MUL: { const float*m_=(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]*=m_[m]; break; }
case PER_ROW_SUB: { const float*m_=(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=m_[m]-ab[m]; break; }
case PER_ROW_SUBF:{ const float*m_=(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]-m_[m]; break; }
// per-col fuse ops degenerate to a single broadcast value at NR=1
case PER_COL_MIN: { float v=*(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]<v?ab[m]:v; break; }
case PER_COL_MAX: { float v=*(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]>v?ab[m]:v; break; }
case PER_COL_ADD: { float v=*(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]+=v; break; }
case PER_COL_MUL: { float v=*(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]*=v; break; }
case PER_COL_SUB: { float v=*(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=v-ab[m]; break; }
case PER_COL_SUBF:{ float v=*(const float*)s->f0; for(long m=0;m<MR;m++) ab[m]=ab[m]-v; break; }
default:
// LeakyRelu / QScale / RoundingShiftRight / ShiftLeft excluded by CAN_FUSE.
return 1;
}
}
}