// vim: ft=arm

    .arm
    .text
    .global armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}
    .type armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}, %function

armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}:

    pld     [r0]
    push    { r4-r12 }
    vpush   { q4-q7 }

{% include "dispatcher.j2" %}

.add_mat_mul:

    cmp     r3, #0
    beq     .non_linear_loop

    mov     r1, r4 // packed A ptr
    pld     [r3]
    pld     [r5]

    pld     [r1, #128]
    pld     [r1, #192]
    pld     [r1, #256]
    pld     [r1, #320]
    pld     [r1, #384]
    pld     [r1, #448]
    pld     [r1, #512]

.packed_packed_loop_1:
    pld     [r5]                           // packed B ptr

    vldmia          r1!, { q0-q1 }
    vldmia          r5!, { s30 }

    vmla.f32        q8, q0, d15[0]
    vmla.f32        q9, q1, d15[0]

    subs            r3, r3, #1
    bne .packed_packed_loop_1

    b   .non_linear_loop

{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
{% set mr = 8 %}{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
{% set mr = 8 %}{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}

.add_unicast:
    {% for reg in range(0, 4) %}
        vld1.f32    d{{reg}}[0], [ r3 ], r4
        vld1.f32    d{{reg}}[1], [ r3 ], r4
    {% endfor %}
    {% for reg in range(0, 2) %}
        vadd.f32 q{{ reg + 8 }}, q{{ reg + 8 }}, q{{reg}}
    {% endfor %}

    b .non_linear_loop

.add_row_col_products:
    vld1.f32        d0[0], [ r4 ]
    vldmia          r3!, { q4-q5 }

    vmla.f32        q8, q4, d0[0]
    vmla.f32        q9, q5, d0[0]

    b .non_linear_loop

.store:
    // r3, r4 <- ptr, rsc
    cmp     r4, #4
    bne     .store_generic

    vst1.f64    {d16-d19}, [r3]!

    b .non_linear_loop

.store_generic:

    {% for reg in range(16, 20) %}
        vst1.f32    d{{reg}}[0], [r3], r4
        vst1.f32    d{{reg}}[1], [r3], r4
    {% endfor %}

    b .non_linear_loop

.load_tile:
    vldmia          r3!, { q8-q15 }
    b .non_linear_loop

.return:
    vpop        { q4-q7 }
    pop         { r4-r12 }

    bx          lr

