{#
// vim: set syntax=asm :

/* mmm 16 x 1

    zmm0

System V ABI:
    args: rdi, rsi, rdx, rcx, r8, r9
    preserve: rbx, rsp, rbp, r12, r13, r14, r15
    scratch: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11
    return: rax (+rdx)

Windows ABI:
    args: RCX, RDX, R8, R9
    preserve: RBX, RBP, RDI, RSI, RSP, R12, R13, R14, R15, and XMM6-15
    scratch: RAX, RCX, RDX, R8, R9, R10, R11, XMM0-5, and the upper portions of ZMM0-15 and ZMM0-15
    return: rax (+rdx)
*/
#}


{% set size = "16x1" %}{% set suffix = suffix %}{% set G = G %}{% set arch = "avx512" %}{% include "preamble.j2" %}

{{L}}clear:
    vzeroall
    jmp     {{L}}non_linear_loop

{{L}}add_mat_mul:
    mov     rcx,    [rdi + 24]   // B
    mov     rax,    [rdi + 16]   // A

    mov     rbx,    [rdi + 8]    // k
    test    rbx,    rbx
    jz      {{L}}non_linear_loop

	cmp rbx, 8
	jl {{L}}main_loop_packed_packed_tail

{{align}} 16
{{L}}main_loop_packed_packed:
	{% include "1x1/packed_packed_loop1/unroll-4.S.raw" %}

    sub             rbx, 4
	cmp rbx,        4
	jge              {{L}}main_loop_packed_packed

	{% for r in range(1, 4) %}
	   vaddps zmm0, zmm0, zmm{{r}}
	{% endfor %}

    test    rbx, rbx
    jz      {{L}}non_linear_loop

{{align}} 16
{{L}}main_loop_packed_packed_tail:
	{% include "1x1/packed_packed_loop1/avx-512.S.raw" %}

	sub             rbx, 1
    jnz				{{L}}main_loop_packed_packed_tail

    jmp      {{L}}non_linear_loop

{% set from = 0 %}{% set to = 0 %}{% include "f32_scalars.j2" %}
{% set mr = 16 %}{% set from = 0 %}{% set to = 0 %}{% include "f32_per_rows.j2" %}
{% set mr = 16 %}{% set from = 0 %}{% set to = 0 %}{% include "f32_per_cols.j2" %}
{% set from = 0 %}{% set to = 0 %}{% include "avx512_mmm_load_tile.j2" %}

{{L}}add_unicast:
    mov     r10,    [rdi + 8]           // c ptr
    mov     rsi,    [rdi + 16]          // row stride

	cmp rsi, 4
	jne {{L}}add_unicast_generic

	vaddps zmm0, zmm0, [r10]

    jmp    {{L}}non_linear_loop

{{L}}add_unicast_generic:
    mov     eax,    0
    {% for i in range(0, 4) %}
        pinsrd  xmm14, eax, {{i}}
        add     eax,    esi
    {% endfor %}
    {% for i in range(0, 4) %}
        pinsrd  xmm15, eax, {{i}}
        add     eax,    esi
    {% endfor %}
    {% for i in range(0, 4) %}
        pinsrd  xmm12, eax, {{i}}
        add     eax,    esi
    {% endfor %}
    {% for i in range(0, 4) %}
        pinsrd  xmm13, eax, {{i}}
        add     eax,    esi
    {% endfor %}
    vperm2f128      ymm14, ymm14, ymm15, 32
    vperm2f128      ymm13, ymm12, ymm13, 32
    vinsertf32x8    zmm14, zmm14, ymm13, 1

    kxnorw          k1, k1, k1
    vgatherdps      zmm12{k1}, [r10 + zmm14]
    vaddps          zmm0, zmm0, zmm12

    jmp    {{L}}non_linear_loop

{{L}}add_row_col_products:
    mov             rax, [ rdi + 8 ]
    mov             rbx, [ rdi + 16 ]

    vbroadcastss    zmm14, dword ptr [rbx]

{% for i in range(0, 1) %}
    vmovups         zmm12,  [rax + {{ i * 64 }}]
    vfmadd231ps     zmm{{i}}, zmm12, zmm14
{% endfor %}
    jmp    {{L}}non_linear_loop

{{L}}store:
    mov     r8,     [rdi + 8]           // c ptr
    mov     rsi,    [rdi + 16]          // row stride

    cmp     rsi, 4
    jne      {{L}}store_noncontiguous

	test r8, 63
	jnz {{L}}store_unaligned

    vmovaps [r8], zmm0
    jmp     {{L}}non_linear_loop

{{L}}store_unaligned:
	vmovups [r8], zmm0
    jmp     {{L}}non_linear_loop

{{L}}store_noncontiguous:
    {% for quarter in range(0, 4) %}
        vextractf32x4 xmm8, zmm0, {{quarter}}
        {% for row in range(0, 4) %}
            vextractps  dword ptr [r8], xmm8, {{row}}
            add         r8, rsi
        {% endfor %}
    {% endfor %}
    jmp     {{L}}non_linear_loop

{% set size = "16x1" %}{% set suffix = suffix %}{% set G = G %}{% set L = L %}{% set arch = "avx512" %}{% include "postamble.j2" %}
