Add native self-hosted instance connection to fluxer_desktop
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
This commit is contained in:
+200
@@ -0,0 +1,200 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0]
|
||||
// q8[1]
|
||||
// q8[2]
|
||||
// q8[3]
|
||||
//
|
||||
// ....
|
||||
//
|
||||
// q15[0]
|
||||
// q15[1]
|
||||
// q15[2]
|
||||
// q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
pld [r1, #128]
|
||||
pld [r1, #192]
|
||||
pld [r1, #256]
|
||||
pld [r1, #320]
|
||||
pld [r1, #384]
|
||||
pld [r1, #448]
|
||||
pld [r1, #512]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r1, #32]
|
||||
vldr d5, [r1, #40]
|
||||
vldr d6, [r1, #48]
|
||||
vldr d7, [r1, #56]
|
||||
vldr d8, [r1, #64]
|
||||
vldr d9, [r1, #72]
|
||||
vldr d10, [r1, #80]
|
||||
vldr d11, [r1, #88]
|
||||
vldr s30, [r5]
|
||||
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
|
||||
vldr d0, [r1, #96]
|
||||
vldr d1, [r1, #104]
|
||||
vldr d2, [r1, #112]
|
||||
vldr d3, [r1, #120]
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
add r1, #128
|
||||
add r5, #4
|
||||
|
||||
{% elsif core == "cortexa9" %}
|
||||
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4-d7}, [r1]!
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
vld1.64 {d8-d11}, [r1]!
|
||||
vld1.f32 d15[0], [r5]!
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
{% else %}
|
||||
|
||||
vldmia r1!, { q0-q3 }
|
||||
vldmia r5!, { s30 }
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vldmia r1!, { q0-q1 }
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
vldmia r1!, { q2-q3 }
|
||||
|
||||
vmla.f32 q12, q0, d15[0]
|
||||
vmla.f32 q13, q1, d15[0]
|
||||
|
||||
vmla.f32 q14, q2, d15[0]
|
||||
vmla.f32 q15, q3, d15[0]
|
||||
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_f32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_rows.tmpliq" mr:32, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_cols.tmpliq" mr:32, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
{% for reg in (0..15) %}
|
||||
vld1.f32 d{{reg}}[0], [ r3 ], r4
|
||||
vld1.f32 d{{reg}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
{% for reg in (0..7) %}
|
||||
vadd.f32 q{{reg|plus:8}}, q{{reg|plus:8}}, q{{reg}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vld1.f32 d0[0], [ r4 ]
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q8, q4, d0[0]
|
||||
vmla.f32 q9, q5, d0[0]
|
||||
|
||||
vmla.f32 q10, q6, d0[0]
|
||||
vmla.f32 q11, q7, d0[0]
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q12, q4, d0[0]
|
||||
vmla.f32 q13, q5, d0[0]
|
||||
|
||||
vmla.f32 q14, q6, d0[0]
|
||||
vmla.f32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4 <- ptr, rsc
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
vst1.f64 {d16-d19}, [r3]!
|
||||
vst1.f64 {d20-d23}, [r3]!
|
||||
vst1.f64 {d24-d27}, [r3]!
|
||||
vst1.f64 {d28-d31}, [r3]!
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
vst1.f32 d{{reg}}[0], [r3], r4
|
||||
vst1.f32 d{{reg}}[1], [r3], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating q0, q1 with q2, q3
|
||||
// packed B buffering (2x4 values): alternating q4 with q5
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}:
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
pld [r5] // packed B ptr
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
{% elsif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, { q0, q1}
|
||||
vldmia r5!, { q2 }
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q8, q0, d4[0]
|
||||
vmla.f32 q9, q1, d4[0]
|
||||
|
||||
vmla.f32 q10, q0, d4[1]
|
||||
vmla.f32 q11, q1, d4[1]
|
||||
|
||||
vmla.f32 q12, q0, d5[0]
|
||||
vmla.f32 q13, q1, d5[0]
|
||||
|
||||
vmla.f32 q14, q0, d5[1]
|
||||
vmla.f32 q15, q1, d5[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #16
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_f32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_rows.tmpliq" mr:8, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_cols.tmpliq" mr:8, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
{% for col in (0..3) %}
|
||||
mov r2, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{col | times: 4 | plus: reg | plus : 16}}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.f32 q8, q0, d8[0]
|
||||
vmla.f32 q9, q1, d8[0]
|
||||
|
||||
vmla.f32 q10, q0, d8[1]
|
||||
vmla.f32 q11, q1, d8[1]
|
||||
|
||||
vmla.f32 q12, q0, d9[0]
|
||||
vmla.f32 q13, q1, d9[0]
|
||||
|
||||
vmla.f32 q14, q0, d9[1]
|
||||
vmla.f32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3,r4,r5 are c,rsc,csc
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 16}}[0], [ r8 ], r4
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 16}}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
vldr d6, [r5, #16]
|
||||
{% elsif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5, d6}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, {q0-q1}
|
||||
vldmia r5!, {d4-d6}
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #24
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_f32_scalars.tmpliq" from:4, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_rows.tmpliq" mr:8, from:4, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_cols.tmpliq" mr:8, from:4, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
{% for col in (0..5) %}
|
||||
mov r2, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{col | times: 4 | plus: reg | plus : 8}}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { d4, d5, d6 }
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
{% for col in (0..5) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vst1.64 d{{col| times: 4 | plus: 8 | plus: reg}}, [ r8 ]!
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
{% for col in (0..5) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 8}}[0], [ r8 ], r4
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 8}}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_min", op:"vmin.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_max", op:"vmax.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_mul", op:"vmul.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_add", op:"vadd.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub", op:"vsub.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsub.f32", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_min", op:"vmin.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_max", op:"vmax.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_mul", op:"vmul.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_add", op:"vadd.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub", op:"vsub.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsub.f32", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_min", op:"vmin.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_max", op:"vmax.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_mul", op:"vmul.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_add", op:"vadd.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub", op:"vsub.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub_flipped", op:"vsub.f32", from:from, to:to, flipped:true%}
|
||||
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
.q_scale:
|
||||
b .unsupported
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: q8..q16
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_32x1_{{suffix}}
|
||||
.type armv7neon_mmm_i32_32x1_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_32x1_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r7]
|
||||
|
||||
.packed_packed:
|
||||
|
||||
.packed_packed_loop_1:
|
||||
vldmia r1!, { q4-q5 }
|
||||
|
||||
vld1.8 { d0[0] }, [ r5 ]!
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmovl.s8 q1, d8
|
||||
vmlal.s16 q8, d2, d0[0]
|
||||
vmlal.s16 q9, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d9
|
||||
vmlal.s16 q10, d2, d0[0]
|
||||
vmlal.s16 q11, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d10
|
||||
vmlal.s16 q12, d2, d0[0]
|
||||
vmlal.s16 q13, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d11
|
||||
vmlal.s16 q14, d2, d0[0]
|
||||
vmlal.s16 q15, d3, d0[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_rows.tmpliq" mr:32, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_cols.tmpliq" mr:32, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
vld1.s8 d0[0], [ r3 ], r4
|
||||
vld1.s8 d0[1], [ r3 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
{% for reg in (16..31) %}
|
||||
vld1.s32 d0[0], [ r3 ], r4
|
||||
vld1.s32 d0[1], [ r3 ], r4
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldm r3, { s0 }
|
||||
|
||||
vldmia r4!, { q4-q7 }
|
||||
|
||||
vmla.s32 q8, q4, d0[0]
|
||||
vmla.s32 q9, q5, d0[0]
|
||||
|
||||
vmla.s32 q10, q6, d0[0]
|
||||
vmla.s32 q11, q7, d0[0]
|
||||
|
||||
vldmia r4!, { q4-q7 }
|
||||
|
||||
vmla.s32 q12, q4, d0[0]
|
||||
vmla.s32 q13, q5, d0[0]
|
||||
|
||||
vmla.s32 q14, q6, d0[0]
|
||||
vmla.s32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.tmpliq" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in (8..15) %}
|
||||
vmovn.s32 d{{reg | times: 2}}, q{{reg}}
|
||||
vmovn.s16 d{{reg | times: 2}}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for reg in (8..15) %}
|
||||
{%capture d%}{{reg | times: 2 }}{%endcapture%}
|
||||
vst1.s8 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for reg in (8..15) %}
|
||||
{%capture d%}{{reg | times: 2}}{%endcapture%}
|
||||
vst1.s32 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s32 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s32 d{{d|plus:1}}[0], [ r3 ], r4
|
||||
vst1.s32 d{{d|plus:1}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_8x4_{{suffix}}
|
||||
.type armv7neon_mmm_i32_8x4_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_8x4_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
cmp r3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.packed_packed_loop_4:
|
||||
pld [r1, #64]
|
||||
pld [r5, #64]
|
||||
|
||||
// q2: d4 -> d4,d5 A even cols (from r1)
|
||||
// q3: d6 -> d6,d7 A odd cols (from r1)
|
||||
// q0: s0 -> d0 : B even lines (from r5)
|
||||
// q1: s4 -> d2 : B odd lines (from r5)
|
||||
|
||||
// 0
|
||||
vldmia r1!, { d4 }
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 1
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vldmia r1!, { d4 }
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
// 2
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 3
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
sub r3, r3, #4
|
||||
cmp r3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
vldmia r1!, { s0, s1 }
|
||||
vmovl.s8 q0, d0
|
||||
vldmia r5!, { s4 }
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d0, d2[0]
|
||||
vmlal.s16 q9, d1, d2[0]
|
||||
|
||||
vmlal.s16 q10, d0, d2[1]
|
||||
vmlal.s16 q11, d1, d2[1]
|
||||
|
||||
vmlal.s16 q12, d0, d2[2]
|
||||
vmlal.s16 q13, d1, d2[2]
|
||||
|
||||
vmlal.s16 q14, d0, d2[3]
|
||||
vmlal.s16 q15, d1, d2[3]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_rows.tmpliq" mr:8, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_cols.tmpliq" mr:8, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.s8 d0[0], [ r8 ], r4
|
||||
vld1.s8 d0[1], [ r8 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{col | times: 4 | plus: reg | plus : 16}}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.s32 d0[0], [ r8 ], r4
|
||||
vld1.s32 d0[1], [ r8 ], r4
|
||||
vadd.i32 d{{col | times: 4 | plus: reg | plus : 16}}, d0
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.s32 q8, q0, d8[0]
|
||||
vmla.s32 q9, q1, d8[0]
|
||||
|
||||
vmla.s32 q10, q0, d8[1]
|
||||
vmla.s32 q11, q1, d8[1]
|
||||
|
||||
vmla.s32 q12, q0, d9[0]
|
||||
vmla.s32 q13, q1, d9[0]
|
||||
|
||||
vmla.s32 q14, q0, d9[1]
|
||||
vmla.s32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.tmpliq" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in (8..15) %}
|
||||
vmovn.s32 d{{reg | times: 2}}, q{{reg}}
|
||||
vmovn.s16 d{{reg | times: 2}}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..1) %}
|
||||
{%capture d%}{{col | times: 2 | plus: reg | times: 2 | plus: 16}}{%endcapture%}
|
||||
vst1.s8 d{{d}}[0], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
{% for lane in (0..1) %}
|
||||
vst1.s32 d{{col | times: 4 | plus: reg | plus: 16}}[{{lane}}], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_min", op:"vmin.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_max", op:"vmax.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_mul", op:"vmul.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_add", op:"vadd.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub", op:"vsub.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsub.s32", mr:mr, from:from, to:to, flipped:true%}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_min", op:"vmin.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_max", op:"vmax.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_mul", op:"vmul.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_add", op:"vadd.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub", op:"vsub.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsub.s32", mr:mr, from:from, to:to, flipped:true%}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_min", op:"vmin.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_max", op:"vmax.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_mul", op:"vmul.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_add", op:"vadd.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub", op:"vsub.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub_flipped", op:"vsub.s32", from:from, to:to, flipped:true%}
|
||||
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.q_scale:
|
||||
ldm r0, { r4, r5, r6, r7 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q0, r7 // q0 <- multiplier
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
vmovl.s32 q1, d2
|
||||
|
||||
add r5, #32
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- -(shift + 32)
|
||||
vmovl.s32 q2, d4
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_scale_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_scale_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_scale_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_scale_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_scale_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_scale_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_scale_rounding_zero:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_away:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_minus_inf:
|
||||
{% for q in (8..15) %}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_plus_inf:
|
||||
{% for q in (8..15) %}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_even:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q3, q3, q1
|
||||
vsub.s64 q4, q4, q1
|
||||
vadd.s64 q5, q3
|
||||
vadd.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_odd:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q5, q3
|
||||
vsub.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
ldm r0, { r4, r5 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
{% for q in (8..15) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Shift
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr:
|
||||
ldm r0, { r4, r5, r6 } // fixme params are already loaded by disp.
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_shr_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_shr_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_shr_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_shr_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_shr_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_shr_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_shr_rounding_zero:
|
||||
// return signum(x) * ((abs(x) - 1) >>r shift )
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vsub.s32 q{{q}}, q1 // Substract 1 to abs(x)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -((abs(x) - 1) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_away:
|
||||
// return signum(x) * (abs(x) >>r shift )
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -(abs(x) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_minus_inf:
|
||||
// return -(-x >>r shift)
|
||||
{% for q in (8..15) %}
|
||||
vneg.s32 q3, q{{q}} // Compute -x
|
||||
vqrshl.s32 q3, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q{{q}}, q3 // Compute -(-x >>r shift)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_plus_inf:
|
||||
// return x >>r shift
|
||||
{% for q in (8..15) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_even:
|
||||
// If (x >> shift) is odd -> (x - 0) >>r shift
|
||||
// If (x >> shift) is even -> (x - 1) >>r shift
|
||||
{% for q in (8..15) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x is odd
|
||||
vsub.s32 q5, q4, q1 // If (x >> shift) is odd 0 else -1
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If (x >> shift) is odd (x - 0) else (x - 1)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_odd:
|
||||
// If (x >> shift) is even -> (x - 0) >>r shift
|
||||
// If (x >> shift) is odd -> (x - 1) >>r shift
|
||||
{% for q in (8..15) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x >> shift is odd
|
||||
vneg.s32 q5, q4 // If x is odd -1 else 0
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If x is odd (x - 1) else (x - 0)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
|
||||
{% capture mr_over_4 %}{{ mr | divided_by: 4}}{%endcapture%}
|
||||
{% capture mr_over_4_min_1 %}{{ mr | divided_by: 4 | minus: 1}}{%endcapture%}
|
||||
|
||||
{%capture cols%}{{to | plus: 1| minus:from| divided_by:mr_over_4}}{%endcapture%}
|
||||
{%capture cols_min_1%}{{to | plus: 1| minus:from| divided_by:mr_over_4|minus:1}}{%endcapture%}
|
||||
|
||||
{% if cols == "1" %}
|
||||
vld1.f32 d0[0], [ r3 ]
|
||||
{% else %}
|
||||
{%capture cols_over_2_minus_1%}{{cols | divided_by:2 | minus:1}}{%endcapture%}
|
||||
{% for c in (0..cols_over_2_minus_1) %}
|
||||
vldmia r3!, { d{{c}} }
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% for right in (0..cols_min_1) %}
|
||||
vdup.f32 q3, d{{right|divided_by:2}}[{{right| modulo:2}}]
|
||||
{% for down in (0..mr_over_4_min_1) %}
|
||||
{%capture acc%}{{mr_over_4|times:right|plus:from|plus:down}}{%endcapture%}
|
||||
{% if flipped %}
|
||||
{{op}} q{{acc}}, q{{acc}}, q3
|
||||
{% else %}
|
||||
{{op}} q{{acc}}, q3, q{{acc}}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
|
||||
{% capture mr_over_4 %}{{ mr | divided_by: 4}}{%endcapture%}
|
||||
{% capture mr_over_4_min_1 %}{{ mr | divided_by: 4 | minus: 1}}{%endcapture%}
|
||||
|
||||
{% for reg in (0..mr_over_4_min_1) %}
|
||||
vldmia r3!, { q{{reg}} }
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%}
|
||||
{{op}} q{{acc}}, q{{acc}}, q{{other}}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%}
|
||||
{{op}} q{{acc}}, q{{other}}, q{{acc}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% if flipped %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} q{{reg}}, q{{reg}}, q0
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} q{{reg}}, q0, q{{reg}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
b .non_linear_loop
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_prefetch_{{suffix}}
|
||||
.type armv7neon_prefetch_{{suffix}}, %function
|
||||
|
||||
armv7neon_prefetch_{{suffix}}:
|
||||
loop:
|
||||
pld [r0]
|
||||
pld [r0, #32]
|
||||
pld [r0, #64]
|
||||
pld [r0, #96]
|
||||
pld [r0, #128]
|
||||
pld [r0, #160]
|
||||
pld [r0, #192]
|
||||
pld [r0, #224]
|
||||
add r0, r0, #256
|
||||
cmp r0, r1
|
||||
blt loop
|
||||
|
||||
bx lr
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_sigmoid_f32_4n_{{suffix}}
|
||||
.type armv7neon_sigmoid_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_sigmoid_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vdup.32 q11, d6[1]
|
||||
vdup.32 q12, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
vmla.f32 q11, q5, q8
|
||||
vmla.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -18.6 // low
|
||||
.float 18.6 // high
|
||||
.float -4.433153405e-18 // alpha_13
|
||||
.float 1.169974371e-14
|
||||
|
||||
.float -1.875289645e-11
|
||||
.float 4.257889523e-8
|
||||
.float 0.00004811817576
|
||||
.float 0.008163842030
|
||||
|
||||
.float 0.2499999971
|
||||
.float 3.922935744e-6 // beta_6
|
||||
.float 0.001524872358
|
||||
.float 0.1159886749
|
||||
|
||||
.float 1.0
|
||||
.float 0.5 //
|
||||
.float 0.0 // padding
|
||||
.float 0.0
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_tanh_f32_4n_{{suffix}}
|
||||
.type armv7neon_tanh_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_tanh_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
vmul.f32 q11, q5, q8
|
||||
vmul.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -8.9 // low
|
||||
.float 8.9 // high
|
||||
.float -8.488492677e-14 // alpha_13
|
||||
.float 5.277853000e-11
|
||||
|
||||
.float -2.022500419e-8
|
||||
.float 0.00001115424833
|
||||
.float 0.003103950131
|
||||
.float 0.1308400453
|
||||
|
||||
.float 0.9999999934
|
||||
.float 0.0002546136580 // beta_6
|
||||
.float 0.02449515379
|
||||
.float 0.4641733162
|
||||
|
||||
.float 1.0
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
|
||||
.non_linear_loop_entry:
|
||||
sub r0, #20
|
||||
|
||||
.non_linear_loop:
|
||||
add r0, #20
|
||||
ldm r0, { r2, r3, r4, r5, r6 }
|
||||
|
||||
cmp r2, #{{ jump_table | size }}
|
||||
movgt r2, #{{ jump_table | size }}
|
||||
cmp r2, #0
|
||||
movlt r2, #{{ jump_table | size }}
|
||||
|
||||
add pc, pc, r2, LSL#2
|
||||
nop // pc in Rn above is start of the add instruction + 8, hence a nop is needed
|
||||
// This is A32 asm, for T32/Thump2 use nop.w and b.w to avoid problems.
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
|
||||
.unsupported:
|
||||
mov r0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
.clear:
|
||||
{% for r in (4..15) %}
|
||||
veor q{{r}}, q{{r}}, q{{r}}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
Reference in New Issue
Block a user