Add native self-hosted instance connection to fluxer_desktop
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
This commit is contained in:
+204
@@ -0,0 +1,204 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0]
|
||||
// q8[1]
|
||||
// q8[2]
|
||||
// q8[3]
|
||||
//
|
||||
// ....
|
||||
//
|
||||
// q15[0]
|
||||
// q15[1]
|
||||
// q15[2]
|
||||
// q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
pld [r1, #128]
|
||||
pld [r1, #192]
|
||||
pld [r1, #256]
|
||||
pld [r1, #320]
|
||||
pld [r1, #384]
|
||||
pld [r1, #448]
|
||||
pld [r1, #512]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r1, #32]
|
||||
vldr d5, [r1, #40]
|
||||
vldr d6, [r1, #48]
|
||||
vldr d7, [r1, #56]
|
||||
vldr d8, [r1, #64]
|
||||
vldr d9, [r1, #72]
|
||||
vldr d10, [r1, #80]
|
||||
vldr d11, [r1, #88]
|
||||
vldr s30, [r5]
|
||||
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
|
||||
vldr d0, [r1, #96]
|
||||
vldr d1, [r1, #104]
|
||||
vldr d2, [r1, #112]
|
||||
vldr d3, [r1, #120]
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
add r1, #128
|
||||
add r5, #4
|
||||
|
||||
{% elif core == "cortexa9" %}
|
||||
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4-d7}, [r1]!
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
vld1.64 {d8-d11}, [r1]!
|
||||
vld1.f32 d15[0], [r5]!
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
{% else %}
|
||||
|
||||
vldmia r1!, { q0-q3 }
|
||||
vldmia r5!, { s30 }
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vldmia r1!, { q0-q1 }
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
vldmia r1!, { q2-q3 }
|
||||
|
||||
vmla.f32 q12, q0, d15[0]
|
||||
vmla.f32 q13, q1, d15[0]
|
||||
|
||||
vmla.f32 q14, q2, d15[0]
|
||||
vmla.f32 q15, q3, d15[0]
|
||||
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
{% for reg in range(0, 16) %}
|
||||
vld1.f32 d{{reg}}[0], [ r3 ], r4
|
||||
vld1.f32 d{{reg}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
{% for reg in range(0, 8) %}
|
||||
vadd.f32 q{{ reg + 8 }}, q{{ reg + 8 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vld1.f32 d0[0], [ r4 ]
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q8, q4, d0[0]
|
||||
vmla.f32 q9, q5, d0[0]
|
||||
|
||||
vmla.f32 q10, q6, d0[0]
|
||||
vmla.f32 q11, q7, d0[0]
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q12, q4, d0[0]
|
||||
vmla.f32 q13, q5, d0[0]
|
||||
|
||||
vmla.f32 q14, q6, d0[0]
|
||||
vmla.f32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4 <- ptr, rsc
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
vst1.f64 {d16-d19}, [r3]!
|
||||
vst1.f64 {d20-d23}, [r3]!
|
||||
vst1.f64 {d24-d27}, [r3]!
|
||||
vst1.f64 {d28-d31}, [r3]!
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
vst1.f32 d{{reg}}[0], [r3], r4
|
||||
vst1.f32 d{{reg}}[1], [r3], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
pld [r1, #128]
|
||||
pld [r1, #192]
|
||||
pld [r1, #256]
|
||||
pld [r1, #320]
|
||||
pld [r1, #384]
|
||||
pld [r1, #448]
|
||||
pld [r1, #512]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
vldmia r1!, { q0-q1 }
|
||||
vldmia r5!, { s30 }
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.f32 d{{reg}}[0], [ r3 ], r4
|
||||
vld1.f32 d{{reg}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
{% for reg in range(0, 2) %}
|
||||
vadd.f32 q{{ reg + 8 }}, q{{ reg + 8 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vld1.f32 d0[0], [ r4 ]
|
||||
vldmia r3!, { q4-q5 }
|
||||
|
||||
vmla.f32 q8, q4, d0[0]
|
||||
vmla.f32 q9, q5, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4 <- ptr, rsc
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
vst1.f64 {d16-d19}, [r3]!
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
{% for reg in range(16, 20) %}
|
||||
vst1.f32 d{{reg}}[0], [r3], r4
|
||||
vst1.f32 d{{reg}}[1], [r3], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating q0, q1 with q2, q3
|
||||
// packed B buffering (2x4 values): alternating q4 with q5
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}:
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
pld [r5] // packed B ptr
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
{% elif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, { q0, q1}
|
||||
vldmia r5!, { q2 }
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q8, q0, d4[0]
|
||||
vmla.f32 q9, q1, d4[0]
|
||||
|
||||
vmla.f32 q10, q0, d4[1]
|
||||
vmla.f32 q11, q1, d4[1]
|
||||
|
||||
vmla.f32 q12, q0, d5[0]
|
||||
vmla.f32 q13, q1, d5[0]
|
||||
|
||||
vmla.f32 q14, q0, d5[1]
|
||||
vmla.f32 q15, q1, d5[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #16
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
{% for col in range(0, 4) %}
|
||||
mov r2, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{ col * 4 + reg + 16 }}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.f32 q8, q0, d8[0]
|
||||
vmla.f32 q9, q1, d8[0]
|
||||
|
||||
vmla.f32 q10, q0, d8[1]
|
||||
vmla.f32 q11, q1, d8[1]
|
||||
|
||||
vmla.f32 q12, q0, d9[0]
|
||||
vmla.f32 q13, q1, d9[0]
|
||||
|
||||
vmla.f32 q14, q0, d9[1]
|
||||
vmla.f32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3,r4,r5 are c,rsc,csc
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vst1.f32 d{{ col * 4 + reg + 16 }}[0], [ r8 ], r4
|
||||
vst1.f32 d{{ col * 4 + reg + 16 }}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
vldr d6, [r5, #16]
|
||||
{% elif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5, d6}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, {q0-q1}
|
||||
vldmia r5!, {d4-d6}
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #24
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 4 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 4 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 4 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
{% for col in range(0, 6) %}
|
||||
mov r2, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{ col * 4 + reg + 8 }}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { d4, d5, d6 }
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
{% for col in range(0, 6) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vst1.64 d{{ col * 4 + 8 + reg }}, [ r8 ]!
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
{% for col in range(0, 6) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vst1.f32 d{{ col * 4 + reg + 8 }}[0], [ r8 ], r4
|
||||
vst1.f32 d{{ col * 4 + reg + 8 }}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q4-q7 }
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col("per_col_min", "vmin.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_max", "vmax.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_mul", "vmul.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_add", "vadd.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub", "vsub.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub_flipped", "vsub.f32", mr, from, to, flipped=true) }}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row("per_row_min", "vmin.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_max", "vmax.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_mul", "vmul.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_add", "vadd.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub", "vsub.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub_flipped", "vsub.f32", mr, from, to, flipped=true) }}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar("scalar_min", "vmin.f32", from, to) }}
|
||||
{{ scalar("scalar_max", "vmax.f32", from, to) }}
|
||||
{{ scalar("scalar_mul", "vmul.f32", from, to) }}
|
||||
{{ scalar("scalar_add", "vadd.f32", from, to) }}
|
||||
{{ scalar("scalar_sub", "vsub.f32", from, to) }}
|
||||
{{ scalar("scalar_sub_flipped", "vsub.f32", from, to, flipped=true) }}
|
||||
|
||||
.leaky_relu:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% for reg in range(from, to + 1) %}
|
||||
vmul.f32 q2, q{{reg}}, q0
|
||||
vcgt.f32 q1, q{{reg}}, 0
|
||||
vbsl q1, q{{reg}}, q2
|
||||
vmov q{{reg}}, q1
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
.q_scale:
|
||||
b .unsupported
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: q8..q16
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_32x1_{{suffix}}
|
||||
.type armv7neon_mmm_i32_32x1_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_32x1_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
// r3 r4 r5 r6
|
||||
// k a b packing
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r7]
|
||||
|
||||
cmp r6, #1
|
||||
beq .packed_packed_i8i8
|
||||
|
||||
.packed_packed:
|
||||
|
||||
.packed_packed_loop_1:
|
||||
vldmia r1!, { q4-q7 }
|
||||
|
||||
vld1.32 { d0[0] }, [ r5 ]!
|
||||
|
||||
vmla.s32 q8, q4, d0[0]
|
||||
|
||||
vldmia r1!, { q1-q4 }
|
||||
|
||||
vmla.s32 q9, q5, d0[0]
|
||||
vmla.s32 q10, q6, d0[0]
|
||||
vmla.s32 q11, q7, d0[0]
|
||||
|
||||
vmla.s32 q12, q1, d0[0]
|
||||
vmla.s32 q13, q2, d0[0]
|
||||
|
||||
vmla.s32 q14, q3, d0[0]
|
||||
vmla.s32 q15, q4, d0[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_i8i8:
|
||||
|
||||
.packed_packed_loop_i8i8_1:
|
||||
vldmia r1!, { q4-q5 }
|
||||
|
||||
vld1.8 { d0[0] }, [ r5 ]!
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmovl.s8 q1, d8
|
||||
vmlal.s16 q8, d2, d0[0]
|
||||
vmlal.s16 q9, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d9
|
||||
vmlal.s16 q10, d2, d0[0]
|
||||
vmlal.s16 q11, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d10
|
||||
vmlal.s16 q12, d2, d0[0]
|
||||
vmlal.s16 q13, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d11
|
||||
vmlal.s16 q14, d2, d0[0]
|
||||
vmlal.s16 q15, d3, d0[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_i8i8_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
vld1.s8 d0[0], [ r3 ], r4
|
||||
vld1.s8 d0[1], [ r3 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
{% for reg in range(16, 32) %}
|
||||
vld1.s32 d0[0], [ r3 ], r4
|
||||
vld1.s32 d0[1], [ r3 ], r4
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldm r4, { s0 }
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.s32 q8, q4, d0[0]
|
||||
vmla.s32 q9, q5, d0[0]
|
||||
|
||||
vmla.s32 q10, q6, d0[0]
|
||||
vmla.s32 q11, q7, d0[0]
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.s32 q12, q4, d0[0]
|
||||
vmla.s32 q13, q5, d0[0]
|
||||
|
||||
vmla.s32 q14, q6, d0[0]
|
||||
vmla.s32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.j2" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in range(8, 16) %}
|
||||
vmovn.s32 d{{ reg * 2 }}, q{{reg}}
|
||||
vmovn.s16 d{{ reg * 2 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for reg in range(8, 16) %}
|
||||
{% set d = reg * 2 %}
|
||||
vst1.s8 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for reg in range(8, 16) %}
|
||||
{% set d = reg * 2 %}
|
||||
vst1.s32 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s32 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s32 d{{ d + 1 }}[0], [ r3 ], r4
|
||||
vst1.s32 d{{ d + 1 }}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_8x4_{{suffix}}
|
||||
.type armv7neon_mmm_i32_8x4_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_8x4_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
// r3 r4 r5 r6
|
||||
// k a b packing
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
cmp r6, #1
|
||||
beq .packed_packed_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
vldmia r1!, { q0, q1 }
|
||||
vldmia r5!, { q2 }
|
||||
|
||||
vmla.s32 q8, q0, d4[0]
|
||||
vmla.s32 q9, q1, d4[0]
|
||||
|
||||
vmla.s32 q10, q0, d4[1]
|
||||
vmla.s32 q11, q1, d4[1]
|
||||
|
||||
vmla.s32 q12, q0, d5[0]
|
||||
vmla.s32 q13, q1, d5[0]
|
||||
|
||||
vmla.s32 q14, q0, d5[1]
|
||||
vmla.s32 q15, q1, d5[1]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_i8i8:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
cmp r3, #4
|
||||
blt .packed_packed_loop_i8i8_1
|
||||
|
||||
.packed_packed_loop_i8i8_4:
|
||||
pld [r1, #64]
|
||||
pld [r5, #64]
|
||||
|
||||
// q2: d4 -> d4,d5 A even cols (from r1)
|
||||
// q3: d6 -> d6,d7 A odd cols (from r1)
|
||||
// q0: s0 -> d0 : B even lines (from r5)
|
||||
// q1: s4 -> d2 : B odd lines (from r5)
|
||||
|
||||
// 0
|
||||
vldmia r1!, { d4 }
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 1
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vldmia r1!, { d4 }
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
// 2
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 3
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
sub r3, r3, #4
|
||||
cmp r3, #4
|
||||
bge .packed_packed_loop_i8i8_4
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_i8i8_1:
|
||||
|
||||
vldmia r1!, { s0, s1 }
|
||||
vmovl.s8 q0, d0
|
||||
vldmia r5!, { s4 }
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d0, d2[0]
|
||||
vmlal.s16 q9, d1, d2[0]
|
||||
|
||||
vmlal.s16 q10, d0, d2[1]
|
||||
vmlal.s16 q11, d1, d2[1]
|
||||
|
||||
vmlal.s16 q12, d0, d2[2]
|
||||
vmlal.s16 q13, d1, d2[2]
|
||||
|
||||
vmlal.s16 q14, d0, d2[3]
|
||||
vmlal.s16 q15, d1, d2[3]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_i8i8_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.s8 d0[0], [ r8 ], r4
|
||||
vld1.s8 d0[1], [ r8 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{ col * 4 + reg + 16 }}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.s32 d0[0], [ r8 ], r4
|
||||
vld1.s32 d0[1], [ r8 ], r4
|
||||
vadd.i32 d{{ col * 4 + reg + 16 }}, d0
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.s32 q8, q0, d8[0]
|
||||
vmla.s32 q9, q1, d8[0]
|
||||
|
||||
vmla.s32 q10, q0, d8[1]
|
||||
vmla.s32 q11, q1, d8[1]
|
||||
|
||||
vmla.s32 q12, q0, d9[0]
|
||||
vmla.s32 q13, q1, d9[0]
|
||||
|
||||
vmla.s32 q14, q0, d9[1]
|
||||
vmla.s32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.j2" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in range(8, 16) %}
|
||||
vmovn.s32 d{{ reg * 2 }}, q{{reg}}
|
||||
vmovn.s16 d{{ reg * 2 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 2) %}
|
||||
{% set d = (col * 2 + reg) * 2 + 16 %}
|
||||
vst1.s8 d{{d}}[0], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 2) %}
|
||||
vst1.s32 d{{ col * 4 + reg + 16 }}[{{lane}}], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col("per_col_min", "vmin.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_max", "vmax.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_mul", "vmul.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_add", "vadd.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub", "vsub.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub_flipped", "vsub.s32", mr, from, to, flipped=true) }}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row("per_row_min", "vmin.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_max", "vmax.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_mul", "vmul.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_add", "vadd.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub", "vsub.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub_flipped", "vsub.s32", mr, from, to, flipped=true) }}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar("scalar_min", "vmin.s32", from, to) }}
|
||||
{{ scalar("scalar_max", "vmax.s32", from, to) }}
|
||||
{{ scalar("scalar_mul", "vmul.s32", from, to) }}
|
||||
{{ scalar("scalar_add", "vadd.s32", from, to) }}
|
||||
{{ scalar("scalar_sub", "vsub.s32", from, to) }}
|
||||
{{ scalar("scalar_sub_flipped", "vsub.s32", from, to, flipped=true) }}
|
||||
|
||||
.leaky_relu:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% for reg in range(from, to + 1) %}
|
||||
vmul.s32 q2, q{{reg}}, q0
|
||||
vcgt.s32 q1, q{{reg}}, 0
|
||||
vbsl q1, q{{reg}}, q2
|
||||
vmov q{{reg}}, q1
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.q_scale:
|
||||
ldm r0, { r4, r5, r6, r7 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q0, r7 // q0 <- multiplier
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
vmovl.s32 q1, d2
|
||||
|
||||
add r5, #32
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- -(shift + 32)
|
||||
vmovl.s32 q2, d4
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_scale_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_scale_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_scale_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_scale_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_scale_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_scale_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_scale_rounding_zero:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_away:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_minus_inf:
|
||||
{% for q in range(8, 16) %}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_plus_inf:
|
||||
{% for q in range(8, 16) %}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_even:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q3, q3, q1
|
||||
vsub.s64 q4, q4, q1
|
||||
vadd.s64 q5, q3
|
||||
vadd.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_odd:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q5, q3
|
||||
vsub.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
ldm r0, { r4, r5 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
{% for q in range(8, 16) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Shift
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr:
|
||||
ldm r0, { r4, r5, r6 } // fixme params are already loaded by disp.
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_shr_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_shr_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_shr_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_shr_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_shr_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_shr_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_shr_rounding_zero:
|
||||
// return signum(x) * ((abs(x) - 1) >>r shift )
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vsub.s32 q{{q}}, q1 // Substract 1 to abs(x)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -((abs(x) - 1) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_away:
|
||||
// return signum(x) * (abs(x) >>r shift )
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -(abs(x) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_minus_inf:
|
||||
// return -(-x >>r shift)
|
||||
{% for q in range(8, 16) %}
|
||||
vneg.s32 q3, q{{q}} // Compute -x
|
||||
vqrshl.s32 q3, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q{{q}}, q3 // Compute -(-x >>r shift)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_plus_inf:
|
||||
// return x >>r shift
|
||||
{% for q in range(8, 16) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_even:
|
||||
// If (x >> shift) is odd -> (x - 0) >>r shift
|
||||
// If (x >> shift) is even -> (x - 1) >>r shift
|
||||
{% for q in range(8, 16) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x is odd
|
||||
vsub.s32 q5, q4, q1 // If (x >> shift) is odd 0 else -1
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If (x >> shift) is odd (x - 0) else (x - 1)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_odd:
|
||||
// If (x >> shift) is even -> (x - 0) >>r shift
|
||||
// If (x >> shift) is odd -> (x - 1) >>r shift
|
||||
{% for q in range(8, 16) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x >> shift is odd
|
||||
vneg.s32 q5, q4 // If x is odd -1 else 0
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If x is odd (x - 1) else (x - 0)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
// Jinja2 macros for armv7neon scalar / per-row / per-col operations.
|
||||
// Shared by both f32 and i32 kernel variants.
|
||||
|
||||
{% macro scalar(label, op, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% if flipped %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} q{{reg}}, q{{reg}}, q0
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} q{{reg}}, q0, q{{reg}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_row(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set mr_over_4_min_1 = mr // 4 - 1 %}
|
||||
|
||||
{% for reg in range(0, mr_over_4_min_1 + 1) %}
|
||||
vldmia r3!, { q{{reg}} }
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} q{{acc}}, q{{acc}}, q{{other}}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} q{{acc}}, q{{other}}, q{{acc}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_col(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set mr_over_4_min_1 = mr // 4 - 1 %}
|
||||
|
||||
{% set cols = (to + 1 - from) // mr_over_4 %}
|
||||
{% set cols_min_1 = (to + 1 - from) // mr_over_4 - 1 %}
|
||||
|
||||
{% if cols == 1 %}
|
||||
vld1.f32 d0[0], [ r3 ]
|
||||
{% else %}
|
||||
{% set cols_over_2_minus_1 = cols // 2 - 1 %}
|
||||
{% for c in range(0, cols_over_2_minus_1 + 1) %}
|
||||
vldmia r3!, { d{{c}} }
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% for right in range(0, cols_min_1 + 1) %}
|
||||
vdup.f32 q3, d{{ right // 2 }}[{{ right % 2 }}]
|
||||
{% for down in range(0, mr_over_4_min_1 + 1) %}
|
||||
{% set acc = mr_over_4 * right + from + down %}
|
||||
{% if flipped %}
|
||||
{{op}} q{{acc}}, q{{acc}}, q3
|
||||
{% else %}
|
||||
{{op}} q{{acc}}, q3, q{{acc}}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_prefetch_{{suffix}}
|
||||
.type armv7neon_prefetch_{{suffix}}, %function
|
||||
|
||||
armv7neon_prefetch_{{suffix}}:
|
||||
loop:
|
||||
pld [r0]
|
||||
pld [r0, #32]
|
||||
pld [r0, #64]
|
||||
pld [r0, #96]
|
||||
pld [r0, #128]
|
||||
pld [r0, #160]
|
||||
pld [r0, #192]
|
||||
pld [r0, #224]
|
||||
add r0, r0, #256
|
||||
cmp r0, r1
|
||||
blt loop
|
||||
|
||||
bx lr
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_sigmoid_f32_4n_{{suffix}}
|
||||
.type armv7neon_sigmoid_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_sigmoid_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vdup.32 q11, d6[1]
|
||||
vdup.32 q12, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
vmla.f32 q11, q5, q8
|
||||
vmla.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -18.6 // low
|
||||
.float 18.6 // high
|
||||
.float -4.433153405e-18 // alpha_13
|
||||
.float 1.169974371e-14
|
||||
|
||||
.float -1.875289645e-11
|
||||
.float 4.257889523e-8
|
||||
.float 0.00004811817576
|
||||
.float 0.008163842030
|
||||
|
||||
.float 0.2499999971
|
||||
.float 3.922935744e-6 // beta_6
|
||||
.float 0.001524872358
|
||||
.float 0.1159886749
|
||||
|
||||
.float 1.0
|
||||
.float 0.5 //
|
||||
.float 0.0 // padding
|
||||
.float 0.0
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_tanh_f32_4n_{{suffix}}
|
||||
.type armv7neon_tanh_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_tanh_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
vmul.f32 q11, q5, q8
|
||||
vmul.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -8.9 // low
|
||||
.float 8.9 // high
|
||||
.float -8.488492677e-14 // alpha_13
|
||||
.float 5.277853000e-11
|
||||
|
||||
.float -2.022500419e-8
|
||||
.float 0.00001115424833
|
||||
.float 0.003103950131
|
||||
.float 0.1308400453
|
||||
|
||||
.float 0.9999999934
|
||||
.float 0.0002546136580 // beta_6
|
||||
.float 0.02449515379
|
||||
.float 0.4641733162
|
||||
|
||||
.float 1.0
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
|
||||
.non_linear_loop_entry:
|
||||
sub r0, #20
|
||||
|
||||
.non_linear_loop:
|
||||
add r0, #20
|
||||
ldm r0, { r2, r3, r4, r5, r6 }
|
||||
|
||||
cmp r2, #{{ jump_table | length }}
|
||||
movgt r2, #{{ jump_table | length }}
|
||||
cmp r2, #0
|
||||
movlt r2, #{{ jump_table | length }}
|
||||
|
||||
add pc, pc, r2, LSL#2
|
||||
nop // pc in Rn above is start of the add instruction + 8, hence a nop is needed
|
||||
// This is A32 asm, for T32/Thump2 use nop.w and b.w to avoid problems.
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
|
||||
.unsupported:
|
||||
mov r0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
.clear:
|
||||
{% for r in range(4, 16) %}
|
||||
veor q{{r}}, q{{r}}, q{{r}}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+491
@@ -0,0 +1,491 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armvfpv2_mmm_f32_4x4_{{suffix}}
|
||||
.type armvfpv2_mmm_f32_4x4_{{suffix}}, %function
|
||||
|
||||
// C tile:
|
||||
|
||||
// s16 s20 s24 s28
|
||||
// s17 s21 s25 s29
|
||||
// s18 s22 s26 s30
|
||||
// s19 s23 s27 s31
|
||||
|
||||
// packed A: (2x4) alternating between (s0-s3) and (s4-s7)
|
||||
// packed B: (2x4) alternating between (s8-s11) and (s12-15)
|
||||
|
||||
// all vfp registers in use.
|
||||
|
||||
armvfpv2_mmm_f32_4x4_{{suffix}}:
|
||||
|
||||
/*
|
||||
pld [r1]
|
||||
pld [r1, #8]
|
||||
pld [r2]
|
||||
pld [r2, #8]
|
||||
*/
|
||||
|
||||
push { r4-r12 } // no lr (we're a leaf), no fp. #24 bytes
|
||||
|
||||
ldr r8, [sp, #28]
|
||||
ldr r9, [sp, #24]
|
||||
|
||||
// r8=rsc, r9=csc
|
||||
|
||||
vmrs r6, FPSCR
|
||||
bic r6, r6, #0x00370000
|
||||
vmsr FPSCR, r6
|
||||
|
||||
vpush { s16-s31 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.clear:
|
||||
eor r6, r6
|
||||
vmov s16, r6
|
||||
vmov.f32 s17, s16
|
||||
vmov.f32 s18, s16
|
||||
vmov.f32 s19, s16
|
||||
vmov.f32 s20, s16
|
||||
vmov.f32 s21, s16
|
||||
vmov.f32 s22, s16
|
||||
vmov.f32 s23, s16
|
||||
vmov.f32 s24, s16
|
||||
vmov.f32 s25, s16
|
||||
vmov.f32 s26, s16
|
||||
vmov.f32 s27, s16
|
||||
vmov.f32 s28, s16
|
||||
vmov.f32 s29, s16
|
||||
vmov.f32 s30, s16
|
||||
vmov.f32 s31, s16
|
||||
b .non_linear_loop
|
||||
|
||||
.add_mat_mul:
|
||||
// r3 <- k, r4 <- a, r5 <- b
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
cmp r3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.packed_packed_loop_4:
|
||||
|
||||
// 1
|
||||
vldmia r1!, { s0, s1 }
|
||||
vldmia r5!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r1!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r5!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vldmia r1!, { s4-s7 }
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vldmia r5!, { s12-s15 }
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
// 2
|
||||
vmla.f32 s16, s4, s12
|
||||
vmla.f32 s17, s5, s12
|
||||
vmla.f32 s18, s6, s12
|
||||
vmla.f32 s19, s7, s12
|
||||
|
||||
vldmia r1!, { s0-s3 }
|
||||
|
||||
vmla.f32 s20, s4, s13
|
||||
vmla.f32 s21, s5, s13
|
||||
vmla.f32 s22, s6, s13
|
||||
vmla.f32 s23, s7, s13
|
||||
|
||||
vldmia r5!, { s8-s11 }
|
||||
|
||||
vmla.f32 s24, s4, s14
|
||||
vmla.f32 s25, s5, s14
|
||||
vmla.f32 s26, s6, s14
|
||||
vmla.f32 s27, s7, s14
|
||||
|
||||
vmla.f32 s28, s4, s15
|
||||
vmla.f32 s29, s5, s15
|
||||
vmla.f32 s30, s6, s15
|
||||
vmla.f32 s31, s7, s15
|
||||
|
||||
// 3
|
||||
vmla.f32 s16, s0, s8
|
||||
vmla.f32 s17, s1, s8
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vldmia r1!, { s4-s7 }
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vldmia r5!, { s12-s15 }
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
pld [r1]
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
pld [r6]
|
||||
|
||||
// 4
|
||||
vmla.f32 s16, s4, s12
|
||||
vmla.f32 s17, s5, s12
|
||||
vmla.f32 s18, s6, s12
|
||||
vmla.f32 s19, s7, s12
|
||||
|
||||
vmla.f32 s20, s4, s13
|
||||
vmla.f32 s21, s5, s13
|
||||
vmla.f32 s22, s6, s13
|
||||
vmla.f32 s23, s7, s13
|
||||
|
||||
vmla.f32 s24, s4, s14
|
||||
vmla.f32 s25, s5, s14
|
||||
vmla.f32 s26, s6, s14
|
||||
vmla.f32 s27, s7, s14
|
||||
|
||||
vmla.f32 s28, s4, s15
|
||||
vmla.f32 s29, s5, s15
|
||||
vmla.f32 s30, s6, s15
|
||||
vmla.f32 s31, s7, s15
|
||||
|
||||
sub r3, r3, #4
|
||||
cmp r3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
vldmia r1!, { s0, s1 }
|
||||
vldmia r5!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r1!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r5!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vldr s0, [ r8 ]
|
||||
vadd.f32 s{{ col * 4 + reg + 16 }}, s{{ col * 4 + reg + 16 }}, s0
|
||||
{% if reg < 3 %}
|
||||
add r8, r8, r4
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_min:
|
||||
vmov s0, r3
|
||||
{% for reg in range(16, 32) %}
|
||||
vcmp.f32 s{{reg}}, s0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge s{{reg}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_max:
|
||||
vmov s0, r3
|
||||
{% for reg in range(16, 32) %}
|
||||
vcmp.f32 s{{reg}}, s0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovle s{{reg}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_add:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vadd.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_mul:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vmul.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vsub.f32 s{{s}}, s0, s{{s}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub_flipped:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vsub.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.leaky_relu:
|
||||
vmov s0, r3
|
||||
{% for reg in range(16, 32) %}
|
||||
vmul.f32 s1, s0, s{{reg}}
|
||||
vcmp.f32 s{{reg}}, #0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt s{{reg}}, s1
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_min:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{row}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge {{s}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_max:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{row}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt {{s}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_add:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vadd.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_mul:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vmul.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{row}}, s{{ col * 4 + row + 16 }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub_flipped:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_min:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{col}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge {{s}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_max:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{col}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt {{s}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_add:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vadd.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_mul:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vmul.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{col}}, s{{ col * 4 + row + 16 }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub_flipped:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { s0, s1 }
|
||||
vldmia r4!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r3!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r4!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
fsts s{{ col * 4 + reg + 16 }}, [ r8 ]
|
||||
{% if reg < 3 %}
|
||||
add r8, r8, r4
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { s16-s31 }
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
|
||||
.return:
|
||||
vpop { s16-s31 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
|
||||
.non_linear_loop_entry:
|
||||
sub r0, #20
|
||||
|
||||
.non_linear_loop:
|
||||
add r0, #20
|
||||
ldm r0, { r2, r3, r4, r5, r6 }
|
||||
|
||||
cmp r2, #{{ jump_table | length }}
|
||||
movgt r2, #{{ jump_table | length }}
|
||||
cmp r2, #0
|
||||
movlt r2, #{{ jump_table | length }}
|
||||
|
||||
add pc, pc, r2, LSL#2
|
||||
nop // pc in Rn above is start of the add instruction + 8, hence a nop is needed
|
||||
// This is A32 asm, for T32/Thump2 use nop.w and b.w to avoid problems.
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
.unsupported:
|
||||
mov r0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
Reference in New Issue
Block a user