Add native self-hosted instance connection to fluxer_desktop
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
This commit is contained in:
+81
@@ -0,0 +1,81 @@
|
||||
|
||||
{% macro scalar(label, op, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
add x2, x0, #8
|
||||
ld1 {v0.s}[0], [ x2 ]
|
||||
dup v0.4s, v0.s[0]
|
||||
{% if flipped %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} v{{reg}}.4s, v0.4s, v{{reg}}.4s
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_row(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set mr_over_4_min_1 = mr // 4 - 1 %}
|
||||
|
||||
{% for reg in range(0, mr_over_4_min_1 + 1) %}
|
||||
ldr q{{reg}}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} v{{acc}}.4s, v{{acc}}.4s, v{{other}}.4s
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} v{{acc}}.4s, v{{other}}.4s, v{{acc}}.4s
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_col(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set cols = (to + 1 - from) // mr_over_4 %}
|
||||
|
||||
{% set loads = cols // 4 %}
|
||||
|
||||
{%if cols == 1 %}
|
||||
ld1 {v0.s}[0], [ x2 ]
|
||||
{% elif cols == 3 %}
|
||||
ldr d0, [ x2 ], #8
|
||||
ld1 {v0.s}[2], [ x2 ]
|
||||
{% else %}
|
||||
{% for reg in range(1, loads + 1) %}
|
||||
ldr q{{ reg - 1 }}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
// {{mr}} {{cols}}
|
||||
|
||||
{% for col in range(1, cols + 1) %}
|
||||
dup v3.4s, v{{ (col - 1) // 4 }}.s[{{ (col - 1) % 4 }}]
|
||||
{% for row in range(1, mr_over_4 + 1) %}
|
||||
{% set acc = (col - 1) * mr_over_4 + row - 1 + from %}
|
||||
{% if flipped %}
|
||||
{{op}} v{{acc}}.4s, v{{acc}}.4s, v3.4s
|
||||
{% else %}
|
||||
{{op}} v{{acc}}.4s, v3.4s, v{{acc}}.4s
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr w4, [x1], #4
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr w22, [x2], #4
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr w23, [x2], #4
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr w8, [x1], #4
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr w24, [x2], #4
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr w9, [x1], #4
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
ldr w25, [x2], #4
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
ldr w10, [x1], #4
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ldr w26, [x2], #4
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
ldr w11, [x1], #4
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
ldr w27, [x2], #4
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
ldr w12, [x1], #4
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
ldr w13, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
ldr w14, [x1], #4
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
ldr w15, [x1], #4
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ins v0.s[0], w4
|
||||
ins v1.s[0], w8
|
||||
ins v2.s[0], w12
|
||||
ins v4.s[0], w20
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[1], w5
|
||||
ins v1.s[1], w9
|
||||
ins v2.s[1], w13
|
||||
ins v4.s[1], w21
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[2], w6
|
||||
ins v1.s[2], w10
|
||||
ins v2.s[2], w14
|
||||
ins v4.s[2], w22
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[3], w7
|
||||
ins v1.s[3], w11
|
||||
ins v2.s[3], w15
|
||||
ins v4.s[3], w23
|
||||
ins v5.s[3], w27
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr w4, [x1]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr w5, [x1, #4]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr w6, [x1, #8]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr w22, [x2], #4
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1, #12]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr w23, [x2], #4
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr w8, [x1, #16]
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr w24, [x2], #4
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr w9, [x1, #20]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
ldr w25, [x2], #4
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
ldr w10, [x1, #24]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ldr w26, [x2], #4
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
ldr w11, [x1, #28]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
ldr w27, [x2], #4
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
ldr w12, [x1, #32]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
ldr w13, [x1, #36]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
ldr w14, [x1, #40]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
ldr w15, [x1, #44]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
prfm pldl1keep, [x1, #512]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
add x1, x1, #48
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ins v0.s[0], w4
|
||||
|
||||
ins v1.s[0], w8
|
||||
ins v2.s[0], w12
|
||||
|
||||
ins v4.s[0], w20
|
||||
ins v5.s[0], w24
|
||||
|
||||
ins v0.s[1], w5
|
||||
ins v1.s[1], w9
|
||||
|
||||
ins v2.s[1], w13
|
||||
ins v4.s[1], w21
|
||||
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[2], w6
|
||||
|
||||
ins v1.s[2], w10
|
||||
ins v2.s[2], w14
|
||||
|
||||
ins v4.s[2], w22
|
||||
ins v5.s[2], w26
|
||||
|
||||
ins v0.s[3], w7
|
||||
ins v1.s[3], w11
|
||||
|
||||
ins v2.s[3], w15
|
||||
ins v4.s[3], w23
|
||||
ins v5.s[3], w27
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr x4, [x1]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr x20, [x2]
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1, #8]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr x21, [x2, #8]
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1, #16]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr x22, [x2, #16]
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1, #24]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr x23, [x2, #24]
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr x8, [x1, #32]
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr x9, [x1, #40]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #384]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #448]
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
prfm pldl1keep, [x1, #512]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
add x1, x1, #48
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
add x2, x2, #32
|
||||
|
||||
|
||||
ins v0.d[0], x4
|
||||
ins v2.d[0], x8
|
||||
|
||||
ins v4.d[0], x20
|
||||
ins v5.d[0], x22
|
||||
|
||||
ins v0.d[1], x5
|
||||
ins v2.d[1], x9
|
||||
|
||||
ins v4.d[1], x21
|
||||
ins v1.d[0], x6
|
||||
|
||||
ins v1.d[1], x7
|
||||
|
||||
ins v5.d[1], x23
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s }}, [x1], #48
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
// mul a: v0, v1, v2, b: v4, v5
|
||||
// load a: d3/x23, d6/x26, d7/x27
|
||||
// load b: x4, x5, x6, x7
|
||||
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr d3, [x1], #8
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr x4, [x2], #8
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr x23, [x1], #8
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr x5, [x2], #8
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr x6, [x2], #8
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr x26, [x1], #8
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr x7, [x2], #8
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr d7, [x1], #8
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr x27, [x1], #8
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
|
||||
// ins b: v4 <- x4/x5
|
||||
// ins a: d3/x23, d6/x26, d7/x27
|
||||
|
||||
ins v4.d[0], x4
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ins v4.d[1], x5
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
ins v3.d[1], x23
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
ins v6.d[1], x26
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
ins v7.d[1], x27
|
||||
|
||||
// mul a: v3, v6, v7, b: v4, v5
|
||||
// ins b, v5 <- x6, x7
|
||||
// load a: d0/x20, d1/x21, d2/x22
|
||||
// load b: x4, x5
|
||||
|
||||
fmla v8.4s, v3.4s, v4.s[0]
|
||||
ins v5.d[0], x6
|
||||
fmla v9.4s, v6.4s, v4.s[0]
|
||||
ins v5.d[1], x7
|
||||
fmla v10.4s, v7.4s, v4.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v11.4s, v3.4s, v4.s[1]
|
||||
ldr x4, [x2], #8
|
||||
fmla v12.4s, v6.4s, v4.s[1]
|
||||
ldr x20, [x1], #8
|
||||
fmla v13.4s, v7.4s, v4.s[1]
|
||||
ldr x5, [x2], #8
|
||||
fmla v14.4s, v3.4s, v4.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v15.4s, v6.4s, v4.s[2]
|
||||
ldr x6, [x2], #8
|
||||
fmla v16.4s, v7.4s, v4.s[2]
|
||||
ldr x21, [x1], #8
|
||||
fmla v17.4s, v3.4s, v4.s[3]
|
||||
ldr x7, [x2], #8
|
||||
|
||||
// load b: x6, x7
|
||||
fmla v18.4s, v6.4s, v4.s[3]
|
||||
ldr d2, [x1], #8
|
||||
fmla v19.4s, v7.4s, v4.s[3]
|
||||
ldr x22, [x1], #8
|
||||
fmla v20.4s, v3.4s, v5.s[0]
|
||||
fmla v21.4s, v6.4s, v5.s[0]
|
||||
fmla v22.4s, v7.4s, v5.s[0]
|
||||
fmla v23.4s, v3.4s, v5.s[1]
|
||||
fmla v24.4s, v6.4s, v5.s[1]
|
||||
fmla v25.4s, v7.4s, v5.s[1]
|
||||
|
||||
// ins a: d0/x20, d1/x21, d2/x22
|
||||
fmla v26.4s, v3.4s, v5.s[2]
|
||||
ins v0.d[1], x20
|
||||
fmla v27.4s, v6.4s, v5.s[2]
|
||||
ins v1.d[1], x21
|
||||
fmla v28.4s, v7.4s, v5.s[2]
|
||||
ins v2.d[1], x22
|
||||
|
||||
// ins b: v4 <- x4, x5
|
||||
fmla v29.4s, v3.4s, v5.s[3]
|
||||
ins v4.d[0], x4
|
||||
fmla v30.4s, v6.4s, v5.s[3]
|
||||
ins v4.d[1], x5
|
||||
fmla v31.4s, v7.4s, v5.s[3]
|
||||
|
||||
// ins b: v5 <- x6, x7
|
||||
ins v5.d[0], x6
|
||||
ins v5.d[1], x7
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v8 v11 v14 v17 v20 v23 v26 v29
|
||||
// v9 v12 v15 v18 v21 v24 v27 v30
|
||||
// v10 v13 v16 v19 v22 v25 v28 v31
|
||||
|
||||
// no preservation for v0-v7:
|
||||
// packed A buffering (2x8 values): rotating over v0..v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_12x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_12x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s }, [ x1 ], #48
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop1/ldr_x_preload.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 12 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 12 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8 ] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
{% for col in range(0, 8) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 3) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 3 + 8 + reg }}.4s, v{{ col * 3 + 8 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
{% for col in range(0, 8) %}
|
||||
{% for reg in range(0, 3) %}
|
||||
fmla v{{ col * 3 + 8 + reg }}.4s, v{{reg}}.4s, v{{ col // 4 + 4 }}.s[{{ col % 4 }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(0, 8) %}
|
||||
str q{{ col * 3 + 8 }}, [ x5 ]
|
||||
str q{{ col * 3 + 9 }}, [ x5, #16 ]
|
||||
str q{{ col * 3 + 10 }}, [ x5, #32 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
{% for col in range(0, 8) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 3) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 3 + 8 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x6, [x1, #8]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x7, [x1, #16]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x8, [x1, #24]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr x9, [x1, #32]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x1, #40]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr x11, [x1, #48]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x12, [x1, #56]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr x24, [x2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x25, [x2, #8]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
add x1, x1, #64
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
add x2, x2, #16
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v2.d[0], x9
|
||||
ins v1.d[0], x7
|
||||
ins v3.d[0], x11
|
||||
ins v4.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v2.d[1], x10
|
||||
ins v1.d[1], x8
|
||||
ins v3.d[1], x12
|
||||
ins v4.d[1], x25
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
ld1 {{ v4.4s }}, [ x2 ], #16
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
// mul a: v0, v1, v2, v3 b: v4
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9(d9/x9)
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4(d4/x9)
|
||||
|
||||
fmla v16.4s, v5.4s, v9.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v6.4s, v9.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v7.4s, v9.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v8.4s, v9.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v5.4s, v9.s[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v9.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v7.4s, v9.s[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.4s, v8.4s, v9.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v5.4s, v9.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.4s, v6.4s, v9.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v7.4s, v9.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.4s, v8.4s, v9.s[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.4s, v5.4s, v9.s[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.4s, v6.4s, v9.s[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.4s, v7.4s, v9.s[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.4s, v8.4s, v9.s[3]
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_16x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_16x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s }, [ x2 ], #16
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop1/cortex_a53.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 16 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 16 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 4 + 16 + reg }}.4s, v{{ col * 4 + 16 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ]
|
||||
ld1 { v4.4s }, [ x3 ]
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
str q{{ col * 4 + 16 + 0 }}, [ x5 ]
|
||||
str q{{ col * 4 + 16 + 1 }}, [ x5, #16 ]
|
||||
str q{{ col * 4 + 16 + 2 }}, [ x5, #32 ]
|
||||
str q{{ col * 4 + 16 + 3 }}, [ x5, #48 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 4 + 16 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
// mul a: v0, v1, v2, v3, v4, v5 b: v7
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9(d9/x9)
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4(d4/x9)
|
||||
|
||||
fmla v16.4s, v5.4s, v9.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v6.4s, v9.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v7.4s, v9.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v8.4s, v9.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v5.4s, v9.s[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v9.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v7.4s, v9.s[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.4s, v8.4s, v9.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v5.4s, v9.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.4s, v6.4s, v9.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v7.4s, v9.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.4s, v8.4s, v9.s[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.4s, v5.4s, v9.s[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.4s, v6.4s, v9.s[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.4s, v7.4s, v9.s[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.4s, v8.4s, v9.s[3]
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
ldr x4, [x1]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
ldr x5, [x1, #8]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
ldr x6, [x1, #16]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
ldr x7, [x1, #24]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
ldr x8, [x1, #32]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
ldr x9, [x1, #40]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
ldr x10, [x1, #48]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
ldr x11, [x1, #56]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
ldr x12, [x1, #64]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
ldr x13, [x1, #72]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
ldr x14, [x1, #80]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
ldr x15, [x1, #88]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
ldr x20, [x2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
ldr x21, [x2, #8]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
add x1, x1, #96
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
add x2, x2, #16
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
prfm pldl1keep, [x1, #384]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
prfm pldl1keep, [x1, #448]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
prfm pldl1keep, [x2, #448]
|
||||
|
||||
ins v0.d[0], x4
|
||||
ins v1.d[0], x6
|
||||
ins v2.d[0], x8
|
||||
ins v3.d[0], x10
|
||||
ins v4.d[0], x12
|
||||
ins v5.d[0], x14
|
||||
ins v7.d[0], x20
|
||||
|
||||
ins v0.d[1], x5
|
||||
ins v1.d[1], x7
|
||||
ins v2.d[1], x9
|
||||
ins v3.d[1], x11
|
||||
ins v4.d[1], x13
|
||||
ins v5.d[1], x15
|
||||
ins v7.d[1], x21
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
ldr d0, [x1], #8
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
ldr x20, [x2], #8
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
ldr x21, [x2], #8
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
ldr d1, [x1], #8
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
ldr x7, [x1], #8
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
ldr d2, [x1], #8
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
ldr x9, [x1], #8
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
ins v0.d[1], x5
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
ins v1.d[1], x7
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
ldr d3, [x1], #8
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
ldr x11, [x1], #8
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
ins v2.d[1], x9
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
ldr d4, [x1], #8
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
ldr x14, [x1], #8
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
ins v7.d[0], x20
|
||||
ins v7.d[1], x21
|
||||
|
||||
ins v5.d[0], x14
|
||||
ins v5.d[1], x15
|
||||
|
||||
ins v3.d[1], x11
|
||||
ins v4.d[1], x13
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
ld1 {{ v4.4s, v5.4s }}, [ x1 ], #32
|
||||
ld1 {{ v7.4s }}, [ x2 ], #16
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// x20..x27 are used, callee-preserved
|
||||
|
||||
// C tile regs: v8 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_24x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_24x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s }, [ x1 ], #32
|
||||
ld1 { v7.4s }, [ x2 ], #16
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/cortex_a53.S.raw" %}
|
||||
{% elif core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 24 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 24 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 6) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 6 + 8 + reg }}.4s, v{{ col * 6 + 8 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ], #64
|
||||
ld1 { v7.4s }, [ x3 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ]
|
||||
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
str q{{ col * 6 + 8 + 0 }}, [ x5 ]
|
||||
str q{{ col * 6 + 8 + 1 }}, [ x5, #16 ]
|
||||
str q{{ col * 6 + 8 + 2 }}, [ x5, #32 ]
|
||||
str q{{ col * 6 + 8 + 3 }}, [ x5, #48 ]
|
||||
str q{{ col * 6 + 8 + 4 }}, [ x5, #64 ]
|
||||
str q{{ col * 6 + 8 + 5 }}, [ x5, #80 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 6) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 6 + 8 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_32x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_32x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .q4f16se
|
||||
cmp x4, #2
|
||||
beq .q4f32se
|
||||
cmp x4, #3
|
||||
beq .f16f16
|
||||
cmp x4, #4
|
||||
beq .f32f16
|
||||
cmp x4, #5
|
||||
beq .f16f32
|
||||
|
||||
sub x3, x3, #1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
ld1 { v8.s }[0], [ x2 ], #4
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 8
|
||||
.q40f16_const:
|
||||
.byte 0xc8, 0xc7, 0xc6, 0xc5, 0xc4, 0xc2, 0xc0, 0xbc
|
||||
.byte 0x00, 0x3c, 0x40, 0x42, 0x44, 0x45, 0x46, 0x47
|
||||
|
||||
.q4f16se:
|
||||
adr x4, .q40f16_const
|
||||
movi v15.16b, 15
|
||||
ld1 {v13.16b}, [ x4 ]
|
||||
eor v12.16b, v12.16b, v12.16b
|
||||
|
||||
.q4f16se_outerloop:
|
||||
{% for i in range(0, 8) %}
|
||||
eor v{{ i + 16 }}.16b, v{{ i + 16 }}.16b, v{{ i + 16 }}.16b
|
||||
{% endfor %}
|
||||
mov x4, #32
|
||||
|
||||
.p2align 4
|
||||
.q4f16se_innerloop:
|
||||
ld1 { v10.16b }, [ x1 ], #16
|
||||
ld1 { v11.h }[0], [ x2 ], #2
|
||||
|
||||
and v9.16b, v10.16b, v15.16b
|
||||
ushr v10.16b, v10.16b, 4
|
||||
|
||||
tbl v9.16b, { v13.16b }, v9.16b
|
||||
tbl v10.16b, { v13.16b }, v10.16b
|
||||
|
||||
zip1 v0.16b, v12.16b, v9.16b
|
||||
zip2 v2.16b, v12.16b, v9.16b
|
||||
zip1 v4.16b, v12.16b, v10.16b
|
||||
zip2 v6.16b, v12.16b, v10.16b
|
||||
|
||||
fcvtl v11.4s, v11.4h
|
||||
|
||||
fcvtl2 v1.4s, v0.8h
|
||||
fcvtl2 v3.4s, v2.8h
|
||||
fcvtl2 v5.4s, v4.8h
|
||||
fcvtl2 v7.4s, v6.8h
|
||||
fcvtl v0.4s, v0.4h
|
||||
fcvtl v2.4s, v2.4h
|
||||
fcvtl v4.4s, v4.4h
|
||||
fcvtl v6.4s, v6.4h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 16 }}.4s, v{{i}}.4s, v11.s[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x4, x4, #1
|
||||
bne .q4f16se_innerloop
|
||||
|
||||
// scales
|
||||
ld1 { v0.8h-v3.8h }, [ x1 ], #64
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v7.4s, v1.8h
|
||||
fcvtl v8.4s, v2.4h
|
||||
fcvtl2 v9.4s, v2.8h
|
||||
fcvtl v10.4s, v3.4h
|
||||
fcvtl2 v11.4s, v3.8h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 24 }}.4s, v{{ i + 4 }}.4s, v{{ i + 16 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #32
|
||||
bne .q4f16se_outerloop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q4f32se:
|
||||
adr x4, .q40f16_const
|
||||
movi v15.16b, 15
|
||||
ld1 {v13.16b}, [ x4 ]
|
||||
eor v12.16b, v12.16b, v12.16b
|
||||
|
||||
.q4f32se_outerloop:
|
||||
{% for i in range(0, 8) %}
|
||||
eor v{{ i + 16 }}.16b, v{{ i + 16 }}.16b, v{{ i + 16 }}.16b
|
||||
{% endfor %}
|
||||
mov x4, #32
|
||||
|
||||
.p2align 4
|
||||
.q4f32se_innerloop:
|
||||
ld1 { v10.16b }, [ x1 ], #16
|
||||
ld1 { v11.s }[0], [ x2 ], #4
|
||||
|
||||
and v9.16b, v10.16b, v15.16b
|
||||
ushr v10.16b, v10.16b, 4
|
||||
|
||||
tbl v9.16b, { v13.16b }, v9.16b
|
||||
tbl v10.16b, { v13.16b }, v10.16b
|
||||
|
||||
zip1 v0.16b, v12.16b, v9.16b
|
||||
zip2 v2.16b, v12.16b, v9.16b
|
||||
zip1 v4.16b, v12.16b, v10.16b
|
||||
zip2 v6.16b, v12.16b, v10.16b
|
||||
|
||||
fcvtl2 v1.4s, v0.8h
|
||||
fcvtl2 v3.4s, v2.8h
|
||||
fcvtl2 v5.4s, v4.8h
|
||||
fcvtl2 v7.4s, v6.8h
|
||||
fcvtl v0.4s, v0.4h
|
||||
fcvtl v2.4s, v2.4h
|
||||
fcvtl v4.4s, v4.4h
|
||||
fcvtl v6.4s, v6.4h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 16 }}.4s, v{{i}}.4s, v11.s[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x4, x4, #1
|
||||
bne .q4f32se_innerloop
|
||||
|
||||
// scales
|
||||
ld1 { v0.8h-v3.8h }, [ x1 ], #64
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v7.4s, v1.8h
|
||||
fcvtl v8.4s, v2.4h
|
||||
fcvtl2 v9.4s, v2.8h
|
||||
fcvtl v10.4s, v3.4h
|
||||
fcvtl2 v11.4s, v3.8h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 24 }}.4s, v{{ i + 4 }}.4s, v{{ i + 16 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #32
|
||||
bne .q4f32se_outerloop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f16:
|
||||
sub x3, x3, #1
|
||||
.f16f16_loop:
|
||||
ld1 { v9.h }[0], [ x2 ], #2
|
||||
ld1 { v10.8h-v13.8h }, [ x1 ], #64
|
||||
|
||||
fcvtl v8.4s, v9.4h
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtl v{{ reg * 2 }}.4s, v{{ reg + 10 }}.4h
|
||||
fcvtl2 v{{ reg * 2 + 1 }}.4s, v{{ reg + 10 }}.8h
|
||||
{% endfor %}
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .f16f16_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f32f16:
|
||||
sub x3, x3, #1
|
||||
.f32f16_loop:
|
||||
ld1 { v9.h }[0], [ x2 ], #2
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
fcvtl v8.4s, v9.4h
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .f32f16_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f32:
|
||||
sub x3, x3, #1
|
||||
.f16f32_loop:
|
||||
ld1 { v8.s }[0], [ x2 ], #4
|
||||
ld1 { v10.8h-v13.8h }, [ x1 ], #64
|
||||
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtl v{{ reg * 2 }}.4s, v{{ reg + 10 }}.4h
|
||||
fcvtl2 v{{ reg * 2 + 1 }}.4s, v{{ reg + 10 }}.8h
|
||||
{% endfor %}
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .f16f32_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #4
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.4s-v3.4s}, [x5], #64
|
||||
ld1 {v4.4s-v7.4s}, [x5], #64
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
fadd v{{ r + 24 }}.4s, v{{ r + 24 }}.4s, v{{r}}.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.s}[0], [ x3 ]
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x2 ], #64
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #2
|
||||
beq .store_f16
|
||||
|
||||
cmp x6, #4
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
st1 { v{{reg}}.4s }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_f16:
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtn v{{reg}}.4h, v{{ reg * 2 + 24 }}.4s
|
||||
fcvtn2 v{{reg}}.8h, v{{ reg * 2 + 25 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig_f16
|
||||
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{reg}}.h }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig_f16:
|
||||
|
||||
{% for reg in range(0, 4) %}
|
||||
st1 { v{{reg}}.8h }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// v8..v15 are callee-preserved
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.global {{G}}arm64simd_mmm_f32_32x3_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_32x3_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .f32f16
|
||||
cmp x4, #2
|
||||
beq .f16f32
|
||||
cmp x4, #3
|
||||
beq .f16f16
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
ld1 { v7.4s }, [ x2 ]
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s }, [ x1 ], #48
|
||||
add x2, x2, #12
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s }, [ x1 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.4s, v{{row}}.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f32f16:
|
||||
ld1 { v7.4h }, [ x2 ]
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s }, [ x1 ], #48
|
||||
fcvtl v7.4s, v7.4h
|
||||
add x2, x2, #6
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s }, [ x1 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.4s, v{{row}}.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .f32f16
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f32:
|
||||
ld1 { v7.4s }, [ x2 ]
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x1 ], #64
|
||||
add x2, x2, #12
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v0.4s, v1.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 9 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 10 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 11 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
fcvtl v4.4s, v2.4h
|
||||
fcvtl2 v5.4s, v2.8h
|
||||
fcvtl v6.4s, v3.4h
|
||||
fcvtl2 v1.4s, v3.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 12 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 13 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 14 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 15 }}.4s, v1.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .f16f32
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f16:
|
||||
ld1 { v7.4h }, [ x2 ]
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x1 ], #64
|
||||
add x2, x2, #6
|
||||
|
||||
fcvtl v7.4s, v7.4h
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v0.4s, v1.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 9 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 10 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 11 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
fcvtl v4.4s, v2.4h
|
||||
fcvtl2 v5.4s, v2.8h
|
||||
fcvtl v6.4s, v3.4h
|
||||
fcvtl2 v1.4s, v3.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 12 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 13 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 14 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 15 }}.4s, v1.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .f16f16
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 8) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 8 + 8 + reg }}.4s, v{{ col * 8 + 8 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x2, x3, [x0, #8]
|
||||
|
||||
ld1 { v7.d }[0], [ x3 ], #8
|
||||
ld1 { v7.s }[2], [ x3 ], #4
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s }, [ x2 ], #48
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s }, [ x2 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.4s, v{{row}}.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #2
|
||||
beq .store_f16
|
||||
|
||||
cmp x6, #4
|
||||
beq .store_strides_contig
|
||||
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 8) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 8 + 8 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for r in range(0, 8) %}
|
||||
st1 { v{{ col * 8 + 8 + r }}.4s }, [ x4 ], 16
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_f16:
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig_f16
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtn v{{reg}}.4h, v{{ (col * 4 + reg) * 2 + 8 }}.4s
|
||||
fcvtn2 v{{reg}}.8h, v{{ (col * 4 + reg) * 2 + 9 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{reg}}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
|
||||
{% endfor %}
|
||||
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig_f16:
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtn v{{reg}}.4h, v{{ (col * 4 + reg) * 2 + 8 }}.4s
|
||||
fcvtn2 v{{reg}}.8h, v{{ (col * 4 + reg) * 2 + 9 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
st1 { v{{reg}}.4s }, [ x4 ], #16
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x5, [x1, #128]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr x6, [x1, #136]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x7, [x1, #144]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr x9, [x1, #152]
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x10, [x1, #96]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr x11, [x1, #104]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x12, [x1, #112]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr x13, [x1, #120]
|
||||
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [ x1 ]
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
ldr x14, [x1, #128]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
ldr x15, [x1, #136]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
ldr x20, [x1, #144]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
ldr x21, [x1, #152]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
ldr x22, [x1, #160]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
ldr x23, [x1, #168]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
ldr x24, [x1, #176]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
ldr x25, [x1, #184]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v1.d[0], x7
|
||||
ins v2.d[0], x10
|
||||
ins v3.d[0], x12
|
||||
ins v4.d[0], x14
|
||||
ins v5.d[0], x20
|
||||
ins v6.d[0], x22
|
||||
ins v7.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v1.d[1], x9
|
||||
ins v2.d[1], x11
|
||||
ins v3.d[1], x13
|
||||
ins v4.d[1], x15
|
||||
ins v5.d[1], x21
|
||||
ins v6.d[1], x23
|
||||
ins v7.d[1], x25
|
||||
|
||||
add x1, x1, #192
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
ld1 {{ v9.4s }}, [ x1 ], #16
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
|
||||
fmla v31.4s, v9.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x8, [x2], #8
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr d12, [x1], #8
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x12, [x1], #8
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr d14, [x1], #8
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x14, [x1], #8
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr d15, [x1], #8
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
ldr x15, [x1], #8
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.d[1], x8
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
ins v12.d[1], x12
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
ins v13.d[1], x13
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
ins v14.d[1], x14
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
ins v15.d[1], x15
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
ldr d14, [x1], #8
|
||||
|
||||
fmla v16.4s, v1.4s, v8.s[2]
|
||||
ldr x14, [x1], #8
|
||||
fmla v17.4s, v2.4s, v8.s[2]
|
||||
ldr d15, [x1], #8
|
||||
fmla v18.4s, v3.4s, v8.s[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v19.4s, v4.4s, v8.s[2]
|
||||
|
||||
ld1 {{ v0.4s }}, [x1], #16
|
||||
|
||||
fmla v20.4s, v5.4s, v8.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v8.s[2]
|
||||
ldr x10, [x1], #8
|
||||
|
||||
fmla v22.4s, v7.4s, v8.s[2]
|
||||
|
||||
fmla v23.4s, v9.4s, v8.s[2]
|
||||
ins v13.d[1], x13
|
||||
fmla v24.4s, v10.4s, v8.s[2]
|
||||
ins v14.d[1], x14
|
||||
fmla v25.4s, v11.4s, v8.s[2]
|
||||
ins v15.d[1], x15
|
||||
|
||||
fmla v26.4s, v12.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
fmla v27.4s, v13.4s, v8.s[2]
|
||||
ins v1.d[1], x10
|
||||
fmla v28.4s, v14.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
fmla v29.4s, v15.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
fmla v30.4s, v0.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
fmla v31.4s, v1.4s, v8.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.s[0], v8.s[3]
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
// load a: v9, v10, v11, v12, v13, v14, v15
|
||||
// load a: v0, v1, v2, v3, v4, v4, v6, v7
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v2.4s, v3.4s, v4.4s, v5.4s }}, [x1], #64
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v6.4s, v7.4s }}, [x1], #32
|
||||
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
fmla v16.4s, v1.4s, v8.s[0]
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
fmla v17.4s, v2.4s, v8.s[0]
|
||||
fmla v18.4s, v3.4s, v8.s[0]
|
||||
fmla v19.4s, v4.4s, v8.s[0]
|
||||
|
||||
fmla v20.4s, v5.4s, v8.s[0]
|
||||
fmla v21.4s, v6.4s, v8.s[0]
|
||||
fmla v22.4s, v7.4s, v8.s[0]
|
||||
fmla v23.4s, v9.4s, v8.s[0]
|
||||
|
||||
fmla v24.4s, v10.4s, v8.s[0]
|
||||
fmla v25.4s, v11.4s, v8.s[0]
|
||||
fmla v26.4s, v12.4s, v8.s[0]
|
||||
fmla v27.4s, v13.4s, v8.s[0]
|
||||
fmla v28.4s, v14.4s, v8.s[0]
|
||||
fmla v29.4s, v15.4s, v8.s[0]
|
||||
fmla v30.4s, v0.4s, v8.s[0]
|
||||
fmla v31.4s, v1.4s, v8.s[0]
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_64x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_64x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
sub x3, x3, #1
|
||||
|
||||
|
||||
ld1 { v8.s }[0], [ x2 ], #4
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop1/cortex_a53.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a53" %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% elif core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop2/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
// last loop can't read beyond actual input as it's likely not packed and padded
|
||||
.packed_packed_loop_1_last:
|
||||
ld1 { v9.4s, v10.4s, v11.4s, v12.4s }, [x1], #64
|
||||
ld1 { v13.4s, v14.4s, v15.4s }, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ld1 { v0.4s }, [ x1 ]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #4
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.4s-v3.4s}, [x5], #64
|
||||
ld1 {v4.4s-v7.4s}, [x5], #64
|
||||
ld1 {v8.4s-v11.4s}, [x5], #64
|
||||
ld1 {v12.4s-v15.4s}, [x5], #64
|
||||
|
||||
{% for r in range(0, 16) %}
|
||||
fadd v{{ r + 16 }}.4s, v{{ r + 16 }}.4s, v{{r}}.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.s}[0], [ x3 ]
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
ldr q{{r}}, [x2], #16
|
||||
{% endfor %}
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr q0, [x2], #16
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr q1, [x2], #16
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr q2, [x2], #16
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr q3, [x2], #16
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr q4, [x2], #16
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr q5, [x2], #16
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr q6, [x2], #16
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr q7, [x2], #16
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc$
|
||||
|
||||
cmp x6, #4
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
st1 { v{{reg}}.4s }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
ld1 {{ v2.4s, v3.4s }}, [x1], #32
|
||||
ld1 {{ v6.4s, v7.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
and v0.16b, v2.16b, v2.16b
|
||||
and v1.16b, v3.16b, v3.16b
|
||||
and v4.16b, v6.16b, v6.16b
|
||||
and v5.16b, v7.16b, v7.16b
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v20.4s, v1.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr w22, [x2], #4
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr w8, [x1], #4
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr w23, [x2], #4
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
ldr w9, [x1], #4
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
ldr w24, [x2], #4
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
ldr w10, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
ldr w25, [x2], #4
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ldr w11, [x1], #4
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ldr w26, [x2], #4
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ldr w12, [x1], #4
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ldr w27, [x2], #4
|
||||
|
||||
ins v0.s[0], w5
|
||||
ins v4.s[0], w20
|
||||
ins v1.s[0], w9
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[2], w7
|
||||
ins v4.s[2], w22
|
||||
ins v1.s[2], w11
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[1], w6
|
||||
ins v4.s[1], w21
|
||||
ins v1.s[1], w10
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[3], w8
|
||||
ins v4.s[3], w23
|
||||
ins v1.s[3], w12
|
||||
ins v5.s[3], w27
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr w22, [x2], #4
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr w8, [x1], #4
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr w23, [x2], #4
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
ldr w9, [x1], #4
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
ldr w24, [x2], #4
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
ldr w10, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
ldr w25, [x2], #4
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ldr w11, [x1], #4
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ldr w26, [x2], #4
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ldr w12, [x1], #4
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ldr w27, [x2], #4
|
||||
|
||||
prfm pldl1keep, [x1, #256]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.s[0], w5
|
||||
ins v4.s[0], w20
|
||||
ins v1.s[0], w9
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[2], w7
|
||||
ins v4.s[2], w22
|
||||
ins v1.s[2], w11
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[1], w6
|
||||
ins v4.s[1], w21
|
||||
ins v1.s[1], w10
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[3], w8
|
||||
ins v4.s[3], w23
|
||||
ins v1.s[3], w12
|
||||
ins v5.s[3], w27
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr x11, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x8, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x12, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ins v2.d[0], x5
|
||||
ins v6.d[0], x9
|
||||
ins v3.d[0], x7
|
||||
ins v7.d[0], x11
|
||||
ins v2.d[1], x6
|
||||
ins v6.d[1], x10
|
||||
ins v3.d[1], x8
|
||||
ins v7.d[1], x12
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr x11, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x8, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x12, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #384]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #448]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #448]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v4.d[0], x9
|
||||
ins v1.d[0], x7
|
||||
ins v5.d[0], x11
|
||||
ins v0.d[1], x6
|
||||
ins v4.d[1], x10
|
||||
ins v1.d[1], x8
|
||||
ins v5.d[1], x12
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
ld1 {{ v2.4s, v3.4s }}, [x1], #32
|
||||
ld1 {{ v6.4s, v7.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v2.4s, v6.s[0]
|
||||
fmla v17.4s, v3.4s, v6.s[0]
|
||||
fmla v18.4s, v2.4s, v6.s[1]
|
||||
fmla v19.4s, v3.4s, v6.s[1]
|
||||
fmla v20.4s, v2.4s, v6.s[2]
|
||||
fmla v21.4s, v3.4s, v6.s[2]
|
||||
fmla v22.4s, v2.4s, v6.s[3]
|
||||
fmla v23.4s, v3.4s, v6.s[3]
|
||||
|
||||
fmla v24.4s, v2.4s, v7.s[0]
|
||||
fmla v25.4s, v3.4s, v7.s[0]
|
||||
fmla v26.4s, v2.4s, v7.s[1]
|
||||
fmla v27.4s, v3.4s, v7.s[1]
|
||||
fmla v28.4s, v2.4s, v7.s[2]
|
||||
fmla v29.4s, v3.4s, v7.s[2]
|
||||
fmla v30.4s, v2.4s, v7.s[3]
|
||||
fmla v31.4s, v3.4s, v7.s[3]
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d2, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d6, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr d7, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x8, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ins v2.d[1], x5
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ins v6.d[1], x7
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ins v3.d[1], x6
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ins v7.d[1], x8
|
||||
|
||||
fmla v16.4s, v2.4s, v6.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v3.4s, v6.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v2.4s, v6.s[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v6.s[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.4s, v2.4s, v6.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v3.4s, v6.s[2]
|
||||
ldr d5, [x2], #8
|
||||
fmla v22.4s, v2.4s, v6.s[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.4s, v3.4s, v6.s[3]
|
||||
ldr x8, [x2], #8
|
||||
|
||||
fmla v24.4s, v2.4s, v7.s[0]
|
||||
fmla v25.4s, v3.4s, v7.s[0]
|
||||
fmla v26.4s, v2.4s, v7.s[1]
|
||||
fmla v27.4s, v3.4s, v7.s[1]
|
||||
fmla v28.4s, v2.4s, v7.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v29.4s, v3.4s, v7.s[2]
|
||||
ins v4.d[1], x7
|
||||
fmla v30.4s, v2.4s, v7.s[3]
|
||||
ins v1.d[1], x6
|
||||
fmla v31.4s, v3.4s, v7.s[3]
|
||||
ins v5.d[1], x8
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_8x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_8x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed:
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop1/ldr_x_preload.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
str q{{ col * 2 }}, [ x5 ]
|
||||
str q{{ col * 2 + 1 }}, [ x5, #16 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col(label="per_col_min", op="fmin", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_max", op="fmax", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_mul", op="fmul", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_add", op="fadd", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub", op="fsub", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub_flipped", op="fsub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row(label="per_row_min", op="fmin", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_max", op="fmax", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_mul", op="fmul", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_add", op="fadd", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub", op="fsub", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub_flipped", op="fsub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar(label="scalar_min", op="fmin", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_max", op="fmax", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_mul", op="fmul", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_add", op="fadd", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub", op="fsub", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub_flipped", op="fsub", from=from, to=to, flipped=true) }}
|
||||
|
||||
|
||||
.clear:
|
||||
{% for r in range(from, to + 1) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.leaky_relu:
|
||||
add x2, x0, #8
|
||||
ld1 {v4.s}[0], [ x2 ]
|
||||
dup v4.4s, v4.s[0]
|
||||
|
||||
// bsl cond/dst, then, else
|
||||
// fcmge dst, src, #0.0
|
||||
{% for r in range(from, to + 1) %}
|
||||
fmul v0.4s, v{{r}}.4s, v4.4s
|
||||
fcmge v1.4s, v{{r}}.4s, #0.0
|
||||
bsl v1.16b, v{{r}}.16b, v0.16b
|
||||
and v{{r}}.16b, v1.16b, v1.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_i32_64x1_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_64x1_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .packed_packed_loop_1_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
ld1 {v9.s}[0], [ x2 ], 4
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x1 ], #64
|
||||
{% for reg in range(0, 4) %}
|
||||
mla v{{ reg * 2 + 16 }}.4s, v{{ reg * 2 }}.4s, v9.s[0]
|
||||
mla v{{ reg * 2 + 17 }}.4s, v{{ reg * 2 + 1 }}.4s, v9.s[0]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x1 ], #64
|
||||
{% for reg in range(0, 4) %}
|
||||
mla v{{ reg * 2 + 24 }}.4s, v{{ reg * 2 }}.4s, v9.s[0]
|
||||
mla v{{ reg * 2 + 25 }}.4s, v{{ reg * 2 + 1 }}.4s, v9.s[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1_i8i8:
|
||||
ld1 {v9.b}[0], [ x2 ], 1
|
||||
sshll v9.8h, v9.8b, 0
|
||||
|
||||
ld1 { v0.8b-v3.8b }, [ x1 ], #32
|
||||
ld1 { v4.8b-v7.8b }, [ x1 ], #32
|
||||
|
||||
{% for reg in range(0, 8) %}
|
||||
sshll v10.8h, v{{reg}}.8b, 0
|
||||
smlal v{{ reg * 2 + 16 }}.4s, v10.4h, v9.h[0]
|
||||
smlal2 v{{ reg * 2 + 17 }}.4s, v10.8h, v9.h[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1_i8i8
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
add v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v15.s }[0], [ x3 ]
|
||||
xtn v15.4h, v15.4s
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x2 ], #64
|
||||
|
||||
{% for reg in range(0, 8) %}
|
||||
xtn v{{reg}}.4h, v{{reg}}.4s
|
||||
smlal v{{ reg + 16 }}.4s, v{{reg}}.4h, v15.h[0]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x2 ], #64
|
||||
|
||||
{% for reg in range(0, 8) %}
|
||||
xtn v{{reg}}.4h, v{{reg}}.4s
|
||||
smlal v{{ reg + 24 }}.4s, v{{reg}}.4h, v15.h[0]
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_cols.j2" %}
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.b }[{{ lane * 4 }}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_i32_8x8_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_8x8_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .packed_packed_loop_1_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
mla v16.4s, v0.4s, v4.s[0]
|
||||
mla v17.4s, v1.4s, v4.s[0]
|
||||
mla v18.4s, v0.4s, v4.s[1]
|
||||
mla v19.4s, v1.4s, v4.s[1]
|
||||
|
||||
mla v20.4s, v0.4s, v4.s[2]
|
||||
mla v21.4s, v1.4s, v4.s[2]
|
||||
mla v22.4s, v0.4s, v4.s[3]
|
||||
mla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
mla v24.4s, v0.4s, v5.s[0]
|
||||
mla v25.4s, v1.4s, v5.s[0]
|
||||
mla v26.4s, v0.4s, v5.s[1]
|
||||
mla v27.4s, v1.4s, v5.s[1]
|
||||
|
||||
mla v28.4s, v0.4s, v5.s[2]
|
||||
mla v29.4s, v1.4s, v5.s[2]
|
||||
mla v30.4s, v0.4s, v5.s[3]
|
||||
mla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1_i8i8:
|
||||
|
||||
ld1 { v0.8b }, [ x1 ], #8
|
||||
sshll v0.8h, v0.8b, 0
|
||||
ld1 { v4.8b }, [ x2 ], #8
|
||||
sshll v4.8h, v4.8b, 0
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal2 v17.4s, v0.8h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal2 v19.4s, v0.8h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal2 v21.4s, v0.8h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal2 v23.4s, v0.8h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v4.h[4]
|
||||
smlal2 v25.4s, v0.8h, v4.h[4]
|
||||
smlal v26.4s, v0.4h, v4.h[5]
|
||||
smlal2 v27.4s, v0.8h, v4.h[5]
|
||||
smlal v28.4s, v0.4h, v4.h[6]
|
||||
smlal2 v29.4s, v0.8h, v4.h[6]
|
||||
smlal v30.4s, v0.4h, v4.h[7]
|
||||
smlal2 v31.4s, v0.8h, v4.h[7]
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1_i8i8
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
xtn v0.4h, v0.4s
|
||||
xtn v1.4h, v1.4s
|
||||
xtn v4.4h, v4.4s
|
||||
xtn v5.4h, v5.4s
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal v17.4s, v1.4h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal v19.4s, v1.4h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal v21.4s, v1.4h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal v23.4s, v1.4h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v5.h[0]
|
||||
smlal v25.4s, v1.4h, v5.h[0]
|
||||
smlal v26.4s, v0.4h, v5.h[1]
|
||||
smlal v27.4s, v1.4h, v5.h[1]
|
||||
smlal v28.4s, v0.4h, v5.h[2]
|
||||
smlal v29.4s, v1.4h, v5.h[2]
|
||||
smlal v30.4s, v0.4h, v5.h[3]
|
||||
smlal v31.4s, v1.4h, v5.h[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.j2" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.b }[{{ lane * 4 }}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd+dotprod
|
||||
.global {{G}}arm64simd_mmm_i32_8x8_dot_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_8x8_dot_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .packed_packed_loop_1_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
mla v16.4s, v0.4s, v4.s[0]
|
||||
mla v17.4s, v1.4s, v4.s[0]
|
||||
mla v18.4s, v0.4s, v4.s[1]
|
||||
mla v19.4s, v1.4s, v4.s[1]
|
||||
|
||||
mla v20.4s, v0.4s, v4.s[2]
|
||||
mla v21.4s, v1.4s, v4.s[2]
|
||||
mla v22.4s, v0.4s, v4.s[3]
|
||||
mla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
mla v24.4s, v0.4s, v5.s[0]
|
||||
mla v25.4s, v1.4s, v5.s[0]
|
||||
mla v26.4s, v0.4s, v5.s[1]
|
||||
mla v27.4s, v1.4s, v5.s[1]
|
||||
|
||||
mla v28.4s, v0.4s, v5.s[2]
|
||||
mla v29.4s, v1.4s, v5.s[2]
|
||||
mla v30.4s, v0.4s, v5.s[3]
|
||||
mla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1_i8i8:
|
||||
// PackedI8K4 (K=4-inner, r=8): per 4-K block, A is m0-3 (v0) / m4-7 (v1),
|
||||
// B is n0-3 (v4) / n4-7 (v5), each lane a 4xi8 group. SDOT by-element dots
|
||||
// a B column's 4 K against all 4 m rows of an A half. Same v16..v31 tile
|
||||
// layout as the SMLAL kernel: v[16 + n*2 + m_half] = C[m_half*4..][n].
|
||||
ld1 { v0.16b, v1.16b }, [ x1 ], #32
|
||||
ld1 { v4.16b, v5.16b }, [ x2 ], #32
|
||||
|
||||
sdot v16.4s, v0.16b, v4.4b[0]
|
||||
sdot v17.4s, v1.16b, v4.4b[0]
|
||||
sdot v18.4s, v0.16b, v4.4b[1]
|
||||
sdot v19.4s, v1.16b, v4.4b[1]
|
||||
sdot v20.4s, v0.16b, v4.4b[2]
|
||||
sdot v21.4s, v1.16b, v4.4b[2]
|
||||
sdot v22.4s, v0.16b, v4.4b[3]
|
||||
sdot v23.4s, v1.16b, v4.4b[3]
|
||||
|
||||
sdot v24.4s, v0.16b, v5.4b[0]
|
||||
sdot v25.4s, v1.16b, v5.4b[0]
|
||||
sdot v26.4s, v0.16b, v5.4b[1]
|
||||
sdot v27.4s, v1.16b, v5.4b[1]
|
||||
sdot v28.4s, v0.16b, v5.4b[2]
|
||||
sdot v29.4s, v1.16b, v5.4b[2]
|
||||
sdot v30.4s, v0.16b, v5.4b[3]
|
||||
sdot v31.4s, v1.16b, v5.4b[3]
|
||||
|
||||
subs x3, x3, #4
|
||||
bgt .packed_packed_loop_1_i8i8
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
xtn v0.4h, v0.4s
|
||||
xtn v1.4h, v1.4s
|
||||
xtn v4.4h, v4.4s
|
||||
xtn v5.4h, v5.4s
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal v17.4s, v1.4h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal v19.4s, v1.4h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal v21.4s, v1.4h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal v23.4s, v1.4h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v5.h[0]
|
||||
smlal v25.4s, v1.4h, v5.h[0]
|
||||
smlal v26.4s, v0.4h, v5.h[1]
|
||||
smlal v27.4s, v1.4h, v5.h[1]
|
||||
smlal v28.4s, v0.4h, v5.h[2]
|
||||
smlal v29.4s, v1.4h, v5.h[2]
|
||||
smlal v30.4s, v0.4h, v5.h[3]
|
||||
smlal v31.4s, v1.4h, v5.h[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.j2" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.b }[{{ lane * 4 }}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col(label="per_col_min", op="smin", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_max", op="smax", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_mul", op="mul", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_add", op="add", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub", op="sub", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub_flipped", op="sub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row(label="per_row_min", op="smin", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_max", op="smax", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_mul", op="mul", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_add", op="add", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub", op="sub", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub_flipped", op="sub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar(label="scalar_min", op="smin", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_max", op="smax", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_mul", op="mul", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_add", op="add", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub", op="sub", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub_flipped", op="sub", from=from, to=to, flipped=true) }}
|
||||
|
||||
.clear:
|
||||
{% for r in range(from, to + 1) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.leaky_relu:
|
||||
add x2, x0, #8
|
||||
ld1 {v4.s}[0], [ x2 ]
|
||||
dup v4.4s, v4.s[0]
|
||||
|
||||
// bsl cond/dst, then, else
|
||||
// fcmge dst, src, #0.0
|
||||
{% for r in range(from, to + 1) %}
|
||||
mul v0.4s, v{{r}}.4s, v4.4s
|
||||
cmge v1.4s, v{{r}}.4s, #0
|
||||
bsl v1.16b, v{{r}}.16b, v0.16b
|
||||
and v{{r}}.16b, v1.16b, v1.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
|
||||
// vim: ft=arm
|
||||
|
||||
.q_scale:
|
||||
ldp x5, x6, [x0, #8] // x5: shift, x6: policy
|
||||
add x2, x0, #24
|
||||
ld1r { v2.4s }, [x2] // v2.4s <- multiplier
|
||||
|
||||
mov w3, #1
|
||||
ins v4.d[0], x3
|
||||
dup v4.2d, v4.d[0] // v4.2d <- 1
|
||||
|
||||
add x5, x5, #32 // add 32 to shift
|
||||
neg x5, x5 // broadcast shift
|
||||
ins v1.d[0], x5
|
||||
dup v1.2d, v1.d[0] // v1.2s <- -(shift + 32)
|
||||
|
||||
cmp x6, 1
|
||||
beq .q_scale_rounding_zero
|
||||
cmp x6, 2
|
||||
beq .q_scale_rounding_away
|
||||
cmp x6, 3
|
||||
beq .q_scale_rounding_minus_inf
|
||||
cmp x6, 4
|
||||
beq .q_scale_rounding_plus_inf
|
||||
cmp x6, 5
|
||||
beq .q_scale_rounding_even
|
||||
cmp x6, 6
|
||||
beq .q_scale_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_scale_rounding_zero:
|
||||
// rust: signum * ((abs + nudge2) >> shift
|
||||
// asm: signum * (2*abs - 1) >>r (shift + 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sub v8.2d, v8.2d, v4.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sub v9.2d, v9.2d, v4.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_away: // signum * (abs >> (shift-1) + 1 >> 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_minus_inf: // val >> shift
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sub v8.2d, v8.2d, v4.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sub v9.2d, v9.2d, v4.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_plus_inf: // (val >> shift-1)+1 >>1
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_even: // signum * ((abs >> shift-1) + (abs & 0x1) - 1 >> 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqshl v3.2d, v8.2d, v1.2d // abs >> shift - 1
|
||||
and v3.16b, v3.16b, v4.16b // abs & 0x1
|
||||
sub v3.2d, v3.2d, v4.2d //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
add v8.2d, v8.2d, v3.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sqshl v3.2d, v9.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
sub v3.2d, v3.2d, v4.2d //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
add v9.2d, v9.2d, v3.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_odd: // signum * ((abs >> shift-1) - (abs & 0x1) >> 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqshl v3.2d, v8.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
sub v8.2d, v8.2d, v3.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sqshl v3.2d, v9.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
sub v9.2d, v9.2d, v3.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
ldr x5, [x0, #8] // x5: shift
|
||||
ins v1.s[0], w5
|
||||
dup v1.4s, v1.s[0] // v1.4s <- shift
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr:
|
||||
ldp x5, x6, [x0, #8] // x5: shift, x6: policy
|
||||
|
||||
mov w3, #1
|
||||
ins v4.s[0], w3
|
||||
dup v4.4s, v4.s[0] // v4.4d <- 1
|
||||
|
||||
neg w5, w5 // broadcast shift
|
||||
ins v1.s[0], w5
|
||||
dup v1.4s, v1.s[0] // v1.4s <- -shift
|
||||
|
||||
cmp x6, 1
|
||||
beq .q_shr_rounding_zero
|
||||
cmp x6, 2
|
||||
beq .q_shr_rounding_away
|
||||
cmp x6, 3
|
||||
beq .q_shr_rounding_minus_inf
|
||||
cmp x6, 4
|
||||
beq .q_shr_rounding_plus_inf
|
||||
cmp x6, 5
|
||||
beq .q_shr_rounding_even
|
||||
cmp x6, 6
|
||||
beq .q_shr_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_shr_rounding_zero:
|
||||
// asm: signum * (abs >>r shift)
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
|
||||
sub v{{q}}.4s, v{{q}}.4s, v4.4s
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_away:
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_minus_inf:
|
||||
{% for q in range(16, 32) %}
|
||||
sqneg v{{q}}.4s, v{{q}}.4s
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
sqneg v{{q}}.4s, v{{q}}.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_plus_inf:
|
||||
{% for q in range(16, 32) %}
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_even:
|
||||
// sqrshl is round(+inf), sqshl trauncates
|
||||
// we look at parity of result by truncation: if it's odd, we have nothing more to do, we go towards +inf
|
||||
// if it's even, we need to nudge towards 0 by adding -1
|
||||
// => nudge = (x >>l shift) & 0x1 - 1 (>>l is sqshl)
|
||||
// => result is (x + nudge) >>r shift (with sqrshl)
|
||||
{% for q in range(16, 32) %}
|
||||
sqshl v3.4s, v{{q}}.4s, v1.4s // trunc
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
sub v3.4s, v3.4s, v4.4s
|
||||
add v{{q}}.4s, v{{q}}.4s, v3.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_odd:
|
||||
// here: nudge is -((x >>l shift) & 0x1)
|
||||
{% for q in range(16, 32) %}
|
||||
sqshl v3.4s, v{{q}}.4s, v1.4s // trunc
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
neg v3.4s, v3.4s
|
||||
add v{{q}}.4s, v{{q}}.4s, v3.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.load_tile:
|
||||
ldr x2, [ x0, #8 ]
|
||||
{% for reg in range(from, to + 1) %}
|
||||
ld1 { v{{reg}}.4s }, [ x2 ], #16
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_sigmoid_f32_4n_{{suffix}}
|
||||
{{G}}arm64simd_sigmoid_f32_4n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
dup v5.4s, v0.s[0] // v5 <- low, broadcasted
|
||||
dup v6.4s, v0.s[1] // v6 <- high, broadcasted
|
||||
dup v7.4s, v3.s[1] // v7 <- 0.5, broadcasted
|
||||
|
||||
cmp x1, #16
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmax v17.4s, v17.4s, v5.4s
|
||||
fmax v18.4s, v18.4s, v5.4s
|
||||
fmax v19.4s, v19.4s, v5.4s
|
||||
|
||||
fmin v16.4s, v16.4s, v6.4s
|
||||
fmin v17.4s, v17.4s, v6.4s
|
||||
fmin v18.4s, v18.4s, v6.4s
|
||||
fmin v19.4s, v19.4s, v6.4s // v16 <- x
|
||||
|
||||
fmul v20.4s, v16.4s, v16.4s
|
||||
fmul v21.4s, v17.4s, v17.4s
|
||||
fmul v22.4s, v18.4s, v18.4s
|
||||
fmul v23.4s, v19.4s, v19.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v25.4s, v0.s[3]
|
||||
fmla v25.4s, v21.4s, v0.s[2]
|
||||
dup v26.4s, v0.s[3]
|
||||
fmla v26.4s, v22.4s, v0.s[2]
|
||||
dup v27.4s, v0.s[3]
|
||||
fmla v27.4s, v23.4s, v0.s[2]
|
||||
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[1]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[1]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[1]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[2]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[2]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[2]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[3]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[3]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[3]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
fmul v16.4s, v16.4s, v28.4s
|
||||
fmul v17.4s, v17.4s, v29.4s
|
||||
fmul v18.4s, v18.4s, v30.4s
|
||||
fmul v19.4s, v19.4s, v31.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v25.4s, v2.s[2]
|
||||
fmla v25.4s, v21.4s, v2.s[1]
|
||||
dup v26.4s, v2.s[2]
|
||||
fmla v26.4s, v22.4s, v2.s[1]
|
||||
dup v27.4s, v2.s[2]
|
||||
fmla v27.4s, v23.4s, v2.s[1]
|
||||
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[3]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[3]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[3]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v3.s[0]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v3.s[0]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v3.s[0]
|
||||
fmla v27.4s, v23.4s, v31.4s // v24 denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
fdiv v17.4s, v17.4s, v25.4s
|
||||
fdiv v18.4s, v18.4s, v26.4s
|
||||
fdiv v19.4s, v19.4s, v27.4s
|
||||
|
||||
fadd v16.4s, v16.4s, v7.4s
|
||||
fadd v17.4s, v17.4s, v7.4s
|
||||
fadd v18.4s, v18.4s, v7.4s
|
||||
fadd v19.4s, v19.4s, v7.4s
|
||||
|
||||
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
|
||||
|
||||
subs x1, x1, #16
|
||||
cmp x1, #16
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmin v16.4s, v16.4s, v6.4s // v16 <- x
|
||||
fmul v20.4s, v16.4s, v16.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
fmul v16.4s, v16.4s, v28.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s // v24 <- denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
fadd v16.4s, v16.4s, v7.4s
|
||||
|
||||
st1 { v16.4s }, [x0], #16
|
||||
|
||||
subs x1, x1, #4
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
.float -18.6 // low
|
||||
.float 18.6 // high
|
||||
.float -4.433153405e-18 // alpha_13
|
||||
.float 1.169974371e-14
|
||||
|
||||
.float -1.875289645e-11
|
||||
.float 4.257889523e-8
|
||||
.float 0.00004811817576
|
||||
.float 0.008163842030
|
||||
|
||||
.float 0.2499999971
|
||||
.float 3.922935744e-6 // beta_6
|
||||
.float 0.001524872358
|
||||
.float 0.1159886749
|
||||
|
||||
.float 1.0
|
||||
.float 0.5 //
|
||||
.float 0.0 // padding
|
||||
.float 0.0
|
||||
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_tanh_f32_4n_{{suffix}}
|
||||
{{G}}arm64simd_tanh_f32_4n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
dup v5.4s, v0.s[0] // v5 <- low, broadcasted
|
||||
dup v6.4s, v0.s[1] // v6 <- high, broadcasted
|
||||
|
||||
cmp x1, #16
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmax v17.4s, v17.4s, v5.4s
|
||||
fmax v18.4s, v18.4s, v5.4s
|
||||
fmax v19.4s, v19.4s, v5.4s
|
||||
|
||||
fmin v16.4s, v16.4s, v6.4s
|
||||
fmin v17.4s, v17.4s, v6.4s
|
||||
fmin v18.4s, v18.4s, v6.4s
|
||||
fmin v19.4s, v19.4s, v6.4s // v16 <- x
|
||||
|
||||
fmul v20.4s, v16.4s, v16.4s
|
||||
fmul v21.4s, v17.4s, v17.4s
|
||||
fmul v22.4s, v18.4s, v18.4s
|
||||
fmul v23.4s, v19.4s, v19.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v25.4s, v0.s[3]
|
||||
fmla v25.4s, v21.4s, v0.s[2]
|
||||
dup v26.4s, v0.s[3]
|
||||
fmla v26.4s, v22.4s, v0.s[2]
|
||||
dup v27.4s, v0.s[3]
|
||||
fmla v27.4s, v23.4s, v0.s[2]
|
||||
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[1]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[1]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[1]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[2]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[2]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[2]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[3]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[3]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[3]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
fmul v16.4s, v16.4s, v28.4s
|
||||
fmul v17.4s, v17.4s, v29.4s
|
||||
fmul v18.4s, v18.4s, v30.4s
|
||||
fmul v19.4s, v19.4s, v31.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v25.4s, v2.s[2]
|
||||
fmla v25.4s, v21.4s, v2.s[1]
|
||||
dup v26.4s, v2.s[2]
|
||||
fmla v26.4s, v22.4s, v2.s[1]
|
||||
dup v27.4s, v2.s[2]
|
||||
fmla v27.4s, v23.4s, v2.s[1]
|
||||
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[3]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[3]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[3]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v3.s[0]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v3.s[0]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v3.s[0]
|
||||
fmla v27.4s, v23.4s, v31.4s // v24 denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
fdiv v17.4s, v17.4s, v25.4s
|
||||
fdiv v18.4s, v18.4s, v26.4s
|
||||
fdiv v19.4s, v19.4s, v27.4s
|
||||
|
||||
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
|
||||
|
||||
subs x1, x1, #16
|
||||
cmp x1, #16
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmin v16.4s, v16.4s, v6.4s // v16 <- x
|
||||
fmul v20.4s, v16.4s, v16.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
fmul v16.4s, v16.4s, v28.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s // v24 <- denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
|
||||
st1 { v16.4s }, [x0], #16
|
||||
|
||||
subs x1, x1, #4
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
.float -8.9 // low
|
||||
.float 8.9 // high
|
||||
.float -8.488492677e-14 // alpha_13
|
||||
.float 5.277853000e-11
|
||||
|
||||
.float -2.022500419e-8
|
||||
.float 0.00001115424833
|
||||
.float 0.003103950131
|
||||
.float 0.1308400453
|
||||
|
||||
.float 0.9999999934
|
||||
.float 0.0002546136580 // beta_6
|
||||
.float 0.02449515379
|
||||
.float 0.4641733162
|
||||
|
||||
.float 1.0
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
sub x0, x0, 40
|
||||
|
||||
.non_linear_loop:
|
||||
add x0, x0, 40
|
||||
ldr x2, [x0]
|
||||
|
||||
mov x4, #{{ jump_table | length }}
|
||||
|
||||
cmp x2, #{{ jump_table | length }}
|
||||
csel x2, x2, x4, lt
|
||||
cmp x2, #0
|
||||
csel x2, x4, x2, lt
|
||||
|
||||
adr x3, .jmp_table
|
||||
add x3, x3, x2, LSL#2
|
||||
br x3
|
||||
|
||||
.jmp_table:
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
add x0, x2, #4000
|
||||
b .return
|
||||
|
||||
.unsupported:
|
||||
mov x0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov x0, 0
|
||||
b .return
|
||||
|
||||
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
// Build-time capability probe for the assembler, used by build.rs
|
||||
// (assembler_supports_dotprod). Older binutils — notably the Debian stretch
|
||||
// aarch64 cross-toolchain in CI — predate FEAT_DotProd and cannot assemble
|
||||
// `sdot` even with `.cpu generic+fp+simd+dotprod`. If this file fails to
|
||||
// assemble, build.rs skips the SDOT kernel and the `tract_arm64_dotprod` cfg,
|
||||
// and the runtime falls back to the SMLAL 8x8 i32 kernel. Not linked into
|
||||
// anything.
|
||||
.cpu generic+fp+simd+dotprod
|
||||
.text
|
||||
.globl tract_dotprod_probe
|
||||
tract_dotprod_probe:
|
||||
sdot v0.4s, v1.16b, v2.4b[0]
|
||||
ret
|
||||
Reference in New Issue
Block a user