Add native self-hosted instance connection to fluxer_desktop
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
name = "tract-linalg"
|
||||
version = "0.19.16"
|
||||
authors = ["Mathieu Poumeyrol <kali@zoy.org>"]
|
||||
autobenches = false
|
||||
description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference"
|
||||
readme = "README.md"
|
||||
keywords = [
|
||||
"TensorFlow",
|
||||
"NeuralNetworks",
|
||||
]
|
||||
categories = ["science"]
|
||||
license = "MIT/Apache-2.0"
|
||||
repository = "https://github.com/snipsco/tract"
|
||||
resolver = "1"
|
||||
|
||||
[[bench]]
|
||||
name = "arm64"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mat_vec"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_wavenet_hw"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "conv_for_wavenet_hw"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_inception"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_asr_am"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "sigmoid"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "arm64simd"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "arm32neon"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "packing"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "virtual_im2col"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "x86_64"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[dependencies.derive-new]
|
||||
version = "0.5.9"
|
||||
|
||||
[dependencies.downcast-rs]
|
||||
version = "1.2.0"
|
||||
|
||||
[dependencies.dyn-clone]
|
||||
version = "1.0.4"
|
||||
|
||||
[dependencies.lazy_static]
|
||||
version = "1.4.0"
|
||||
|
||||
[dependencies.log]
|
||||
version = "0.4.14"
|
||||
|
||||
[dependencies.num-traits]
|
||||
version = "0.2.14"
|
||||
|
||||
[dependencies.paste]
|
||||
version = "1.0.5"
|
||||
|
||||
[dependencies.scan_fmt]
|
||||
version = "0.2.6"
|
||||
|
||||
[dependencies.tract-data]
|
||||
version = "=0.19.16"
|
||||
|
||||
[dev-dependencies.criterion]
|
||||
version = "0.4"
|
||||
|
||||
[dev-dependencies.nu-ansi-term]
|
||||
version = "0.46"
|
||||
|
||||
[dev-dependencies.proptest]
|
||||
version = "1.0.0"
|
||||
|
||||
[build-dependencies.cc]
|
||||
version = "1.0.69"
|
||||
|
||||
[build-dependencies.half]
|
||||
version = "2"
|
||||
features = [
|
||||
"std",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[build-dependencies.liquid]
|
||||
version = "0.26"
|
||||
|
||||
[build-dependencies.liquid-core]
|
||||
version = "0.26"
|
||||
|
||||
[build-dependencies.smallvec]
|
||||
version = "1.6.1"
|
||||
|
||||
[build-dependencies.unicode-normalization]
|
||||
version = "0.1.19"
|
||||
|
||||
[build-dependencies.walkdir]
|
||||
version = "2.3.2"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
no_fp16 = []
|
||||
|
||||
[badges.maintenance]
|
||||
status = "actively-developed"
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
[package]
|
||||
name = "tract-linalg"
|
||||
version = "0.19.16"
|
||||
license = "MIT/Apache-2.0"
|
||||
authors = ["Mathieu Poumeyrol <kali@zoy.org>"]
|
||||
description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference"
|
||||
repository = "https://github.com/snipsco/tract"
|
||||
keywords = ["TensorFlow", "NeuralNetworks"]
|
||||
categories = ["science"]
|
||||
autobenches = false
|
||||
edition = "2021"
|
||||
|
||||
[badges]
|
||||
maintenance = { status = "actively-developed" }
|
||||
|
||||
[dependencies]
|
||||
derive-new.workspace = true
|
||||
downcast-rs.workspace = true
|
||||
dyn-clone.workspace = true
|
||||
lazy_static.workspace = true
|
||||
log.workspace = true
|
||||
num-traits.workspace = true
|
||||
paste.workspace = true
|
||||
scan_fmt.workspace = true
|
||||
tract-data = { version = "=0.19.16", path = "../data" }
|
||||
|
||||
[build-dependencies]
|
||||
cc.workspace = true
|
||||
liquid.workspace = true
|
||||
liquid-core.workspace = true
|
||||
unicode-normalization.workspace = true
|
||||
smallvec.workspace = true
|
||||
walkdir.workspace = true
|
||||
half.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
nu-ansi-term.workspace = true
|
||||
proptest.workspace = true
|
||||
|
||||
[features]
|
||||
# This feature is meant to accomodate very restrictive / legacy toolchains that do
|
||||
# have support for fp16 instructions, breaking tract compilation.
|
||||
# It is not meant to be used in other situations, where run-time detection is
|
||||
# preferred.
|
||||
no_fp16 = []
|
||||
default = []
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "arm64"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mat_vec"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_wavenet_hw"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "conv_for_wavenet_hw"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_inception"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_asr_am"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "sigmoid"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "arm64simd"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "arm32neon"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "packing"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "virtual_im2col"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "x86_64"
|
||||
harness = false
|
||||
@@ -0,0 +1,12 @@
|
||||
## License
|
||||
|
||||
Licensed under either of
|
||||
* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
|
||||
* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
|
||||
at your option.
|
||||
|
||||
### Contribution
|
||||
|
||||
Unless you explicitly state otherwise, any contribution intentionally submitted
|
||||
for inclusion in the work by you, as defined in the Apache-2.0 license, shall
|
||||
be dual licensed as above, without any additional terms or conditions.
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,23 @@
|
||||
Permission is hereby granted, free of charge, to any
|
||||
person obtaining a copy of this software and associated
|
||||
documentation files (the "Software"), to deal in the
|
||||
Software without restriction, including without
|
||||
limitation the rights to use, copy, modify, merge,
|
||||
publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software
|
||||
is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
The above copyright notice and this permission notice
|
||||
shall be included in all copies or substantial portions
|
||||
of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
||||
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
|
||||
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
||||
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
|
||||
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
@@ -0,0 +1,27 @@
|
||||
# tract-linalg
|
||||
|
||||
linalg stands for "linear algebra". This is a misnamer. This crates contains
|
||||
low-level, architecture dependant optimisations used by tract-core.
|
||||
|
||||
# Functions
|
||||
|
||||
* MatMatMul: Extended matrix*matrix product:
|
||||
* inspired by Gotoblass and BLIS micro kernel approach
|
||||
* extended for convolution friendly addressing (fused img2col)
|
||||
* fused output pipeline (min, max, and a few more simple, fast ops)
|
||||
* f32*f32 -> f32 (à la sgemm)
|
||||
* i8*i8 -> i32 accumulator -> i32 storage
|
||||
* i8*i8 -> i32 accumulator -> i8 (with channel zeropoint and scale, and re-quantization pipeline)
|
||||
* f32 sigmoid and f32 tanh: at f32 precision, by a rationale function (no exponentiation)
|
||||
* byte-to-byte lookup table
|
||||
|
||||
# Implementations
|
||||
|
||||
| | generic fallback | armv6, vfp | armv7 neon | armv8 simd | x64 FMA
|
||||
|-------------------|--------------------|---------------|-------------------|-------------------|-----------------
|
||||
| MatMatMul f32 | | 4x4 | 8x4 | 8x8 | 16x6
|
||||
| MatMatMul i8->i8 | | | 8x4 | | 8x8
|
||||
| MatMatMul i8->i32 | | | | | 8x8
|
||||
| sigmoid f32 | | | 4n | 4n |
|
||||
| tanh f32 | | | 4n | 4n |
|
||||
| byte lookup | | | | |
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0]
|
||||
// q8[1]
|
||||
// q8[2]
|
||||
// q8[3]
|
||||
//
|
||||
// ....
|
||||
//
|
||||
// q15[0]
|
||||
// q15[1]
|
||||
// q15[2]
|
||||
// q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
pld [r1, #128]
|
||||
pld [r1, #192]
|
||||
pld [r1, #256]
|
||||
pld [r1, #320]
|
||||
pld [r1, #384]
|
||||
pld [r1, #448]
|
||||
pld [r1, #512]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r1, #32]
|
||||
vldr d5, [r1, #40]
|
||||
vldr d6, [r1, #48]
|
||||
vldr d7, [r1, #56]
|
||||
vldr d8, [r1, #64]
|
||||
vldr d9, [r1, #72]
|
||||
vldr d10, [r1, #80]
|
||||
vldr d11, [r1, #88]
|
||||
vldr s30, [r5]
|
||||
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
|
||||
vldr d0, [r1, #96]
|
||||
vldr d1, [r1, #104]
|
||||
vldr d2, [r1, #112]
|
||||
vldr d3, [r1, #120]
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
add r1, #128
|
||||
add r5, #4
|
||||
|
||||
{% elsif core == "cortexa9" %}
|
||||
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4-d7}, [r1]!
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
vld1.64 {d8-d11}, [r1]!
|
||||
vld1.f32 d15[0], [r5]!
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
{% else %}
|
||||
|
||||
vldmia r1!, { q0-q3 }
|
||||
vldmia r5!, { s30 }
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vldmia r1!, { q0-q1 }
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
vldmia r1!, { q2-q3 }
|
||||
|
||||
vmla.f32 q12, q0, d15[0]
|
||||
vmla.f32 q13, q1, d15[0]
|
||||
|
||||
vmla.f32 q14, q2, d15[0]
|
||||
vmla.f32 q15, q3, d15[0]
|
||||
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_f32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_rows.tmpliq" mr:32, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_cols.tmpliq" mr:32, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
{% for reg in (0..15) %}
|
||||
vld1.f32 d{{reg}}[0], [ r3 ], r4
|
||||
vld1.f32 d{{reg}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
{% for reg in (0..7) %}
|
||||
vadd.f32 q{{reg|plus:8}}, q{{reg|plus:8}}, q{{reg}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vld1.f32 d0[0], [ r4 ]
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q8, q4, d0[0]
|
||||
vmla.f32 q9, q5, d0[0]
|
||||
|
||||
vmla.f32 q10, q6, d0[0]
|
||||
vmla.f32 q11, q7, d0[0]
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q12, q4, d0[0]
|
||||
vmla.f32 q13, q5, d0[0]
|
||||
|
||||
vmla.f32 q14, q6, d0[0]
|
||||
vmla.f32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4 <- ptr, rsc
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
vst1.f64 {d16-d19}, [r3]!
|
||||
vst1.f64 {d20-d23}, [r3]!
|
||||
vst1.f64 {d24-d27}, [r3]!
|
||||
vst1.f64 {d28-d31}, [r3]!
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
vst1.f32 d{{reg}}[0], [r3], r4
|
||||
vst1.f32 d{{reg}}[1], [r3], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating q0, q1 with q2, q3
|
||||
// packed B buffering (2x4 values): alternating q4 with q5
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}:
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
pld [r5] // packed B ptr
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
{% elsif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, { q0, q1}
|
||||
vldmia r5!, { q2 }
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q8, q0, d4[0]
|
||||
vmla.f32 q9, q1, d4[0]
|
||||
|
||||
vmla.f32 q10, q0, d4[1]
|
||||
vmla.f32 q11, q1, d4[1]
|
||||
|
||||
vmla.f32 q12, q0, d5[0]
|
||||
vmla.f32 q13, q1, d5[0]
|
||||
|
||||
vmla.f32 q14, q0, d5[1]
|
||||
vmla.f32 q15, q1, d5[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #16
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_f32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_rows.tmpliq" mr:8, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_cols.tmpliq" mr:8, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
{% for col in (0..3) %}
|
||||
mov r2, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{col | times: 4 | plus: reg | plus : 16}}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.f32 q8, q0, d8[0]
|
||||
vmla.f32 q9, q1, d8[0]
|
||||
|
||||
vmla.f32 q10, q0, d8[1]
|
||||
vmla.f32 q11, q1, d8[1]
|
||||
|
||||
vmla.f32 q12, q0, d9[0]
|
||||
vmla.f32 q13, q1, d9[0]
|
||||
|
||||
vmla.f32 q14, q0, d9[1]
|
||||
vmla.f32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3,r4,r5 are c,rsc,csc
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 16}}[0], [ r8 ], r4
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 16}}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
vldr d6, [r5, #16]
|
||||
{% elsif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5, d6}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, {q0-q1}
|
||||
vldmia r5!, {d4-d6}
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #24
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_f32_scalars.tmpliq" from:4, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_rows.tmpliq" mr:8, from:4, to:15 %}
|
||||
{% include "armv7neon_mmm_f32_per_cols.tmpliq" mr:8, from:4, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
{% for col in (0..5) %}
|
||||
mov r2, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{col | times: 4 | plus: reg | plus : 8}}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { d4, d5, d6 }
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
{% for col in (0..5) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vst1.64 d{{col| times: 4 | plus: 8 | plus: reg}}, [ r8 ]!
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
{% for col in (0..5) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 8}}[0], [ r8 ], r4
|
||||
vst1.f32 d{{col | times: 4 | plus: reg | plus : 8}}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_min", op:"vmin.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_max", op:"vmax.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_mul", op:"vmul.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_add", op:"vadd.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub", op:"vsub.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsub.f32", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_min", op:"vmin.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_max", op:"vmax.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_mul", op:"vmul.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_add", op:"vadd.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub", op:"vsub.f32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsub.f32", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_min", op:"vmin.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_max", op:"vmax.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_mul", op:"vmul.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_add", op:"vadd.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub", op:"vsub.f32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub_flipped", op:"vsub.f32", from:from, to:to, flipped:true%}
|
||||
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
.q_scale:
|
||||
b .unsupported
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: q8..q16
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_32x1_{{suffix}}
|
||||
.type armv7neon_mmm_i32_32x1_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_32x1_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r7]
|
||||
|
||||
.packed_packed:
|
||||
|
||||
.packed_packed_loop_1:
|
||||
vldmia r1!, { q4-q5 }
|
||||
|
||||
vld1.8 { d0[0] }, [ r5 ]!
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmovl.s8 q1, d8
|
||||
vmlal.s16 q8, d2, d0[0]
|
||||
vmlal.s16 q9, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d9
|
||||
vmlal.s16 q10, d2, d0[0]
|
||||
vmlal.s16 q11, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d10
|
||||
vmlal.s16 q12, d2, d0[0]
|
||||
vmlal.s16 q13, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d11
|
||||
vmlal.s16 q14, d2, d0[0]
|
||||
vmlal.s16 q15, d3, d0[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_rows.tmpliq" mr:32, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_cols.tmpliq" mr:32, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
vld1.s8 d0[0], [ r3 ], r4
|
||||
vld1.s8 d0[1], [ r3 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
{% for reg in (16..31) %}
|
||||
vld1.s32 d0[0], [ r3 ], r4
|
||||
vld1.s32 d0[1], [ r3 ], r4
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldm r3, { s0 }
|
||||
|
||||
vldmia r4!, { q4-q7 }
|
||||
|
||||
vmla.s32 q8, q4, d0[0]
|
||||
vmla.s32 q9, q5, d0[0]
|
||||
|
||||
vmla.s32 q10, q6, d0[0]
|
||||
vmla.s32 q11, q7, d0[0]
|
||||
|
||||
vldmia r4!, { q4-q7 }
|
||||
|
||||
vmla.s32 q12, q4, d0[0]
|
||||
vmla.s32 q13, q5, d0[0]
|
||||
|
||||
vmla.s32 q14, q6, d0[0]
|
||||
vmla.s32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.tmpliq" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in (8..15) %}
|
||||
vmovn.s32 d{{reg | times: 2}}, q{{reg}}
|
||||
vmovn.s16 d{{reg | times: 2}}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for reg in (8..15) %}
|
||||
{%capture d%}{{reg | times: 2 }}{%endcapture%}
|
||||
vst1.s8 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for reg in (8..15) %}
|
||||
{%capture d%}{{reg | times: 2}}{%endcapture%}
|
||||
vst1.s32 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s32 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s32 d{{d|plus:1}}[0], [ r3 ], r4
|
||||
vst1.s32 d{{d|plus:1}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_8x4_{{suffix}}
|
||||
.type armv7neon_mmm_i32_8x4_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_8x4_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
cmp r3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.packed_packed_loop_4:
|
||||
pld [r1, #64]
|
||||
pld [r5, #64]
|
||||
|
||||
// q2: d4 -> d4,d5 A even cols (from r1)
|
||||
// q3: d6 -> d6,d7 A odd cols (from r1)
|
||||
// q0: s0 -> d0 : B even lines (from r5)
|
||||
// q1: s4 -> d2 : B odd lines (from r5)
|
||||
|
||||
// 0
|
||||
vldmia r1!, { d4 }
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 1
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vldmia r1!, { d4 }
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
// 2
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 3
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
sub r3, r3, #4
|
||||
cmp r3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
vldmia r1!, { s0, s1 }
|
||||
vmovl.s8 q0, d0
|
||||
vldmia r5!, { s4 }
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d0, d2[0]
|
||||
vmlal.s16 q9, d1, d2[0]
|
||||
|
||||
vmlal.s16 q10, d0, d2[1]
|
||||
vmlal.s16 q11, d1, d2[1]
|
||||
|
||||
vmlal.s16 q12, d0, d2[2]
|
||||
vmlal.s16 q13, d1, d2[2]
|
||||
|
||||
vmlal.s16 q14, d0, d2[3]
|
||||
vmlal.s16 q15, d1, d2[3]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scalars.tmpliq" from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_rows.tmpliq" mr:8, from:8, to:15 %}
|
||||
{% include "armv7neon_mmm_i32_per_cols.tmpliq" mr:8, from:8, to:15 %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.s8 d0[0], [ r8 ], r4
|
||||
vld1.s8 d0[1], [ r8 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{col | times: 4 | plus: reg | plus : 16}}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vld1.s32 d0[0], [ r8 ], r4
|
||||
vld1.s32 d0[1], [ r8 ], r4
|
||||
vadd.i32 d{{col | times: 4 | plus: reg | plus : 16}}, d0
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.s32 q8, q0, d8[0]
|
||||
vmla.s32 q9, q1, d8[0]
|
||||
|
||||
vmla.s32 q10, q0, d8[1]
|
||||
vmla.s32 q11, q1, d8[1]
|
||||
|
||||
vmla.s32 q12, q0, d9[0]
|
||||
vmla.s32 q13, q1, d9[0]
|
||||
|
||||
vmla.s32 q14, q0, d9[1]
|
||||
vmla.s32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.tmpliq" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in (8..15) %}
|
||||
vmovn.s32 d{{reg | times: 2}}, q{{reg}}
|
||||
vmovn.s16 d{{reg | times: 2}}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..1) %}
|
||||
{%capture d%}{{col | times: 2 | plus: reg | times: 2 | plus: 16}}{%endcapture%}
|
||||
vst1.s8 d{{d}}[0], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
{% for lane in (0..1) %}
|
||||
vst1.s32 d{{col | times: 4 | plus: reg | plus: 16}}[{{lane}}], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_min", op:"vmin.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_max", op:"vmax.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_mul", op:"vmul.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_add", op:"vadd.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub", op:"vsub.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_col.tmpliq" label:"per_col_sub_flipped", op:"vsub.s32", mr:mr, from:from, to:to, flipped:true%}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_min", op:"vmin.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_max", op:"vmax.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_mul", op:"vmul.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_add", op:"vadd.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub", op:"vsub.s32", mr:mr, from:from, to:to %}
|
||||
{% include "armv7neon_mmm_q_per_row.tmpliq" label:"per_row_sub_flipped", op:"vsub.s32", mr:mr, from:from, to:to, flipped:true%}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_min", op:"vmin.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_max", op:"vmax.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_mul", op:"vmul.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_add", op:"vadd.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub", op:"vsub.s32", from:from, to:to%}
|
||||
{% include "armv7neon_mmm_q_scalar.tmpliq" label:"scalar_sub_flipped", op:"vsub.s32", from:from, to:to, flipped:true%}
|
||||
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.q_scale:
|
||||
ldm r0, { r4, r5, r6, r7 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q0, r7 // q0 <- multiplier
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
vmovl.s32 q1, d2
|
||||
|
||||
add r5, #32
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- -(shift + 32)
|
||||
vmovl.s32 q2, d4
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_scale_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_scale_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_scale_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_scale_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_scale_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_scale_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_scale_rounding_zero:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_away:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_minus_inf:
|
||||
{% for q in (8..15) %}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_plus_inf:
|
||||
{% for q in (8..15) %}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_even:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q3, q3, q1
|
||||
vsub.s64 q4, q4, q1
|
||||
vadd.s64 q5, q3
|
||||
vadd.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_odd:
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{q | times:2}}, d0[0]
|
||||
vqdmull.s32 q6, d{{q | times:2 | plus:1}}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q5, q3
|
||||
vsub.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{q | times:2}}, q5
|
||||
vmovn.s64 d{{q | times:2 | plus: 1}}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
ldm r0, { r4, r5 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
{% for q in (8..15) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Shift
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr:
|
||||
ldm r0, { r4, r5, r6 } // fixme params are already loaded by disp.
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_shr_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_shr_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_shr_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_shr_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_shr_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_shr_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_shr_rounding_zero:
|
||||
// return signum(x) * ((abs(x) - 1) >>r shift )
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vsub.s32 q{{q}}, q1 // Substract 1 to abs(x)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -((abs(x) - 1) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_away:
|
||||
// return signum(x) * (abs(x) >>r shift )
|
||||
{% for q in (8..15) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -(abs(x) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_minus_inf:
|
||||
// return -(-x >>r shift)
|
||||
{% for q in (8..15) %}
|
||||
vneg.s32 q3, q{{q}} // Compute -x
|
||||
vqrshl.s32 q3, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q{{q}}, q3 // Compute -(-x >>r shift)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_plus_inf:
|
||||
// return x >>r shift
|
||||
{% for q in (8..15) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_even:
|
||||
// If (x >> shift) is odd -> (x - 0) >>r shift
|
||||
// If (x >> shift) is even -> (x - 1) >>r shift
|
||||
{% for q in (8..15) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x is odd
|
||||
vsub.s32 q5, q4, q1 // If (x >> shift) is odd 0 else -1
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If (x >> shift) is odd (x - 0) else (x - 1)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_odd:
|
||||
// If (x >> shift) is even -> (x - 0) >>r shift
|
||||
// If (x >> shift) is odd -> (x - 1) >>r shift
|
||||
{% for q in (8..15) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x >> shift is odd
|
||||
vneg.s32 q5, q4 // If x is odd -1 else 0
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If x is odd (x - 1) else (x - 0)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
|
||||
{% capture mr_over_4 %}{{ mr | divided_by: 4}}{%endcapture%}
|
||||
{% capture mr_over_4_min_1 %}{{ mr | divided_by: 4 | minus: 1}}{%endcapture%}
|
||||
|
||||
{%capture cols%}{{to | plus: 1| minus:from| divided_by:mr_over_4}}{%endcapture%}
|
||||
{%capture cols_min_1%}{{to | plus: 1| minus:from| divided_by:mr_over_4|minus:1}}{%endcapture%}
|
||||
|
||||
{% if cols == "1" %}
|
||||
vld1.f32 d0[0], [ r3 ]
|
||||
{% else %}
|
||||
{%capture cols_over_2_minus_1%}{{cols | divided_by:2 | minus:1}}{%endcapture%}
|
||||
{% for c in (0..cols_over_2_minus_1) %}
|
||||
vldmia r3!, { d{{c}} }
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% for right in (0..cols_min_1) %}
|
||||
vdup.f32 q3, d{{right|divided_by:2}}[{{right| modulo:2}}]
|
||||
{% for down in (0..mr_over_4_min_1) %}
|
||||
{%capture acc%}{{mr_over_4|times:right|plus:from|plus:down}}{%endcapture%}
|
||||
{% if flipped %}
|
||||
{{op}} q{{acc}}, q{{acc}}, q3
|
||||
{% else %}
|
||||
{{op}} q{{acc}}, q3, q{{acc}}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
|
||||
{% capture mr_over_4 %}{{ mr | divided_by: 4}}{%endcapture%}
|
||||
{% capture mr_over_4_min_1 %}{{ mr | divided_by: 4 | minus: 1}}{%endcapture%}
|
||||
|
||||
{% for reg in (0..mr_over_4_min_1) %}
|
||||
vldmia r3!, { q{{reg}} }
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%}
|
||||
{{op}} q{{acc}}, q{{acc}}, q{{other}}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%}
|
||||
{{op}} q{{acc}}, q{{other}}, q{{acc}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% if flipped %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} q{{reg}}, q{{reg}}, q0
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} q{{reg}}, q0, q{{reg}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
b .non_linear_loop
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_prefetch_{{suffix}}
|
||||
.type armv7neon_prefetch_{{suffix}}, %function
|
||||
|
||||
armv7neon_prefetch_{{suffix}}:
|
||||
loop:
|
||||
pld [r0]
|
||||
pld [r0, #32]
|
||||
pld [r0, #64]
|
||||
pld [r0, #96]
|
||||
pld [r0, #128]
|
||||
pld [r0, #160]
|
||||
pld [r0, #192]
|
||||
pld [r0, #224]
|
||||
add r0, r0, #256
|
||||
cmp r0, r1
|
||||
blt loop
|
||||
|
||||
bx lr
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_sigmoid_f32_4n_{{suffix}}
|
||||
.type armv7neon_sigmoid_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_sigmoid_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vdup.32 q11, d6[1]
|
||||
vdup.32 q12, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
vmla.f32 q11, q5, q8
|
||||
vmla.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -18.6 // low
|
||||
.float 18.6 // high
|
||||
.float -4.433153405e-18 // alpha_13
|
||||
.float 1.169974371e-14
|
||||
|
||||
.float -1.875289645e-11
|
||||
.float 4.257889523e-8
|
||||
.float 0.00004811817576
|
||||
.float 0.008163842030
|
||||
|
||||
.float 0.2499999971
|
||||
.float 3.922935744e-6 // beta_6
|
||||
.float 0.001524872358
|
||||
.float 0.1159886749
|
||||
|
||||
.float 1.0
|
||||
.float 0.5 //
|
||||
.float 0.0 // padding
|
||||
.float 0.0
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_tanh_f32_4n_{{suffix}}
|
||||
.type armv7neon_tanh_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_tanh_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
vmul.f32 q11, q5, q8
|
||||
vmul.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -8.9 // low
|
||||
.float 8.9 // high
|
||||
.float -8.488492677e-14 // alpha_13
|
||||
.float 5.277853000e-11
|
||||
|
||||
.float -2.022500419e-8
|
||||
.float 0.00001115424833
|
||||
.float 0.003103950131
|
||||
.float 0.1308400453
|
||||
|
||||
.float 0.9999999934
|
||||
.float 0.0002546136580 // beta_6
|
||||
.float 0.02449515379
|
||||
.float 0.4641733162
|
||||
|
||||
.float 1.0
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
|
||||
.non_linear_loop_entry:
|
||||
sub r0, #20
|
||||
|
||||
.non_linear_loop:
|
||||
add r0, #20
|
||||
ldm r0, { r2, r3, r4, r5, r6 }
|
||||
|
||||
cmp r2, #{{ jump_table | size }}
|
||||
movgt r2, #{{ jump_table | size }}
|
||||
cmp r2, #0
|
||||
movlt r2, #{{ jump_table | size }}
|
||||
|
||||
add pc, pc, r2, LSL#2
|
||||
nop // pc in Rn above is start of the add instruction + 8, hence a nop is needed
|
||||
// This is A32 asm, for T32/Thump2 use nop.w and b.w to avoid problems.
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
|
||||
.unsupported:
|
||||
mov r0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
.clear:
|
||||
{% for r in (4..15) %}
|
||||
veor q{{r}}, q{{r}}, q{{r}}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+479
@@ -0,0 +1,479 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armvfpv2_mmm_f32_4x4_{{suffix}}
|
||||
.type armvfpv2_mmm_f32_4x4_{{suffix}}, %function
|
||||
|
||||
// C tile:
|
||||
|
||||
// s16 s20 s24 s28
|
||||
// s17 s21 s25 s29
|
||||
// s18 s22 s26 s30
|
||||
// s19 s23 s27 s31
|
||||
|
||||
// packed A: (2x4) alternating between (s0-s3) and (s4-s7)
|
||||
// packed B: (2x4) alternating between (s8-s11) and (s12-15)
|
||||
|
||||
// all vfp registers in use.
|
||||
|
||||
armvfpv2_mmm_f32_4x4_{{suffix}}:
|
||||
|
||||
/*
|
||||
pld [r1]
|
||||
pld [r1, #8]
|
||||
pld [r2]
|
||||
pld [r2, #8]
|
||||
*/
|
||||
|
||||
push { r4-r12 } // no lr (we're a leaf), no fp. #24 bytes
|
||||
|
||||
ldr r8, [sp, #28]
|
||||
ldr r9, [sp, #24]
|
||||
|
||||
// r8=rsc, r9=csc
|
||||
|
||||
vmrs r6, FPSCR
|
||||
bic r6, r6, #0x00370000
|
||||
vmsr FPSCR, r6
|
||||
|
||||
vpush { s16-s31 }
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.clear:
|
||||
eor r6, r6
|
||||
vmov s16, r6
|
||||
vmov.f32 s17, s16
|
||||
vmov.f32 s18, s16
|
||||
vmov.f32 s19, s16
|
||||
vmov.f32 s20, s16
|
||||
vmov.f32 s21, s16
|
||||
vmov.f32 s22, s16
|
||||
vmov.f32 s23, s16
|
||||
vmov.f32 s24, s16
|
||||
vmov.f32 s25, s16
|
||||
vmov.f32 s26, s16
|
||||
vmov.f32 s27, s16
|
||||
vmov.f32 s28, s16
|
||||
vmov.f32 s29, s16
|
||||
vmov.f32 s30, s16
|
||||
vmov.f32 s31, s16
|
||||
b .non_linear_loop
|
||||
|
||||
.add_mat_mul:
|
||||
// r3 <- k, r4 <- a, r5 <- b
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
cmp r3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.packed_packed_loop_4:
|
||||
|
||||
// 1
|
||||
vldmia r1!, { s0, s1 }
|
||||
vldmia r5!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r1!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r5!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vldmia r1!, { s4-s7 }
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vldmia r5!, { s12-s15 }
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
// 2
|
||||
vmla.f32 s16, s4, s12
|
||||
vmla.f32 s17, s5, s12
|
||||
vmla.f32 s18, s6, s12
|
||||
vmla.f32 s19, s7, s12
|
||||
|
||||
vldmia r1!, { s0-s3 }
|
||||
|
||||
vmla.f32 s20, s4, s13
|
||||
vmla.f32 s21, s5, s13
|
||||
vmla.f32 s22, s6, s13
|
||||
vmla.f32 s23, s7, s13
|
||||
|
||||
vldmia r5!, { s8-s11 }
|
||||
|
||||
vmla.f32 s24, s4, s14
|
||||
vmla.f32 s25, s5, s14
|
||||
vmla.f32 s26, s6, s14
|
||||
vmla.f32 s27, s7, s14
|
||||
|
||||
vmla.f32 s28, s4, s15
|
||||
vmla.f32 s29, s5, s15
|
||||
vmla.f32 s30, s6, s15
|
||||
vmla.f32 s31, s7, s15
|
||||
|
||||
// 3
|
||||
vmla.f32 s16, s0, s8
|
||||
vmla.f32 s17, s1, s8
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vldmia r1!, { s4-s7 }
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vldmia r5!, { s12-s15 }
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
pld [r1]
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
pld [r6]
|
||||
|
||||
// 4
|
||||
vmla.f32 s16, s4, s12
|
||||
vmla.f32 s17, s5, s12
|
||||
vmla.f32 s18, s6, s12
|
||||
vmla.f32 s19, s7, s12
|
||||
|
||||
vmla.f32 s20, s4, s13
|
||||
vmla.f32 s21, s5, s13
|
||||
vmla.f32 s22, s6, s13
|
||||
vmla.f32 s23, s7, s13
|
||||
|
||||
vmla.f32 s24, s4, s14
|
||||
vmla.f32 s25, s5, s14
|
||||
vmla.f32 s26, s6, s14
|
||||
vmla.f32 s27, s7, s14
|
||||
|
||||
vmla.f32 s28, s4, s15
|
||||
vmla.f32 s29, s5, s15
|
||||
vmla.f32 s30, s6, s15
|
||||
vmla.f32 s31, s7, s15
|
||||
|
||||
sub r3, r3, #4
|
||||
cmp r3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
vldmia r1!, { s0, s1 }
|
||||
vldmia r5!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r1!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r5!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
vldr s0, [ r8 ]
|
||||
vadd.f32 s{{col|times:4|plus:reg|plus:16}}, s{{col|times:4|plus:reg|plus:16}}, s0
|
||||
{% if reg < 3 %}
|
||||
add r8, r8, r4
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_min:
|
||||
vmov s0, r3
|
||||
{% for reg in (16..31) %}
|
||||
vcmp.f32 s{{reg}}, s0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge s{{reg}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_max:
|
||||
vmov s0, r3
|
||||
{% for reg in (16..31) %}
|
||||
vcmp.f32 s{{reg}}, s0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovle s{{reg}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_add:
|
||||
vmov s0, r3
|
||||
{% for s in (16..31) %}
|
||||
vadd.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_mul:
|
||||
vmov s0, r3
|
||||
{% for s in (16..31) %}
|
||||
vmul.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub:
|
||||
vmov s0, r3
|
||||
{% for s in (16..31) %}
|
||||
vsub.f32 s{{s}}, s0, s{{s}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub_flipped:
|
||||
vmov s0, r3
|
||||
{% for s in (16..31) %}
|
||||
vsub.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
|
||||
.per_row_min:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
{%capture s%}s{{col|times:4|plus:row|plus:16}}{%endcapture%}
|
||||
vcmp.f32 {{s}}, s{{row}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge {{s}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_max:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
{%capture s%}s{{col|times:4|plus:row|plus:16}}{%endcapture%}
|
||||
vcmp.f32 {{s}}, s{{row}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt {{s}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_add:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vadd.f32 s{{col|times:4|plus:row|plus:16}}, s{{col|times:4|plus:row|plus:16}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_mul:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vmul.f32 s{{col|times:4|plus:row|plus:16}}, s{{col|times:4|plus:row|plus:16}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vsub.f32 s{{col|times:4|plus:row|plus:16}}, s{{row}}, s{{col|times:4|plus:row|plus:16}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub_flipped:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vsub.f32 s{{col|times:4|plus:row|plus:16}}, s{{col|times:4|plus:row|plus:16}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_min:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
{%capture s%}s{{col|times:4|plus:row|plus:16}}{%endcapture%}
|
||||
vcmp.f32 {{s}}, s{{col}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge {{s}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_max:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
{%capture s%}s{{col|times:4|plus:row|plus:16}}{%endcapture%}
|
||||
vcmp.f32 {{s}}, s{{col}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt {{s}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_add:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vadd.f32 s{{col|times:4|plus:row|plus:16}}, s{{col|times:4|plus:row|plus:16}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_mul:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vmul.f32 s{{col|times:4|plus:row|plus:16}}, s{{col|times:4|plus:row|plus:16}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vsub.f32 s{{col|times:4|plus:row|plus:16}}, s{{col}}, s{{col|times:4|plus:row|plus:16}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub_flipped:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in (0..3) %}
|
||||
{% for col in (0..3) %}
|
||||
vsub.f32 s{{col|times:4|plus:row|plus:16}}, s{{col|times:4|plus:row|plus:16}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { s0, s1 }
|
||||
vldmia r4!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r3!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r4!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
{% for col in (0..3) %}
|
||||
mov r8, r3
|
||||
{% for reg in (0..3) %}
|
||||
fsts s{{col|times:4|plus:reg|plus:16}}, [ r8 ]
|
||||
{% if reg < 3 %}
|
||||
add r8, r8, r4
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
|
||||
.return:
|
||||
vpop { s16-s31 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
|
||||
.non_linear_loop_entry:
|
||||
sub r0, #20
|
||||
|
||||
.non_linear_loop:
|
||||
add r0, #20
|
||||
ldm r0, { r2, r3, r4, r5, r6 }
|
||||
|
||||
cmp r2, #{{ jump_table | size }}
|
||||
movgt r2, #{{ jump_table | size }}
|
||||
cmp r2, #0
|
||||
movlt r2, #{{ jump_table | size }}
|
||||
|
||||
add pc, pc, r2, LSL#2
|
||||
nop // pc in Rn above is start of the add instruction + 8, hence a nop is needed
|
||||
// This is A32 asm, for T32/Thump2 use nop.w and b.w to avoid problems.
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
.unsupported:
|
||||
mov r0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% capture mr_over_8 %}{{ mr | divided_by: 8}}{%endcapture%}
|
||||
{% capture cols%}{{to | plus: 1| minus:from| divided_by:mr_over_8}}{%endcapture%}
|
||||
|
||||
{% capture loads %}{{cols | divided_by: 8}}{% endcapture %}
|
||||
|
||||
{%if cols == "1" %}
|
||||
ld1 {v0.h}[0], [ x2 ]
|
||||
{% elsif cols == "4" %}
|
||||
ldr d0, [ x2 ]
|
||||
{% else %}
|
||||
{% for reg in (1..loads) %}
|
||||
ldr q{{reg |minus:1}}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
// mr:{{mr}} {{ loads }} {{cols}}
|
||||
|
||||
{% for col in (1..cols) %}
|
||||
dup v3.8h, v{{col| minus: 1|divided_by:8}}.h[{{col| minus: 1|modulo:8}}]
|
||||
{% for row in (1..mr_over_8) %}
|
||||
{% capture acc %}{{ col|minus:1|times:mr_over_8|plus:row|minus:1|plus:from }}{% endcapture %}
|
||||
{% if flipped %}
|
||||
{{op}} v{{acc}}.8h, v{{acc}}.8h, v3.8h
|
||||
{% else %}
|
||||
{{op}} v{{acc}}.8h, v3.8h, v{{acc}}.8h
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% capture mr_over_8 %}{{ mr | divided_by: 8 }}{%endcapture%}
|
||||
{% capture mr_over_8_min_1 %}{{ mr | divided_by: 8 | minus: 1 }}{%endcapture%}
|
||||
|
||||
{% for reg in (0..mr_over_8_min_1) %}
|
||||
ldr q{{reg}}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_8}}{%endcapture%}
|
||||
{{op}} v{{acc}}.8h, v{{acc}}.8h, v{{other}}.8h
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_8}}{%endcapture%}
|
||||
{{op}} v{{acc}}.8h, v{{other}}.8h, v{{acc}}.8h
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
add x2, x0, #8
|
||||
ld1 {v0.h}[0], [ x2 ]
|
||||
dup v0.8h, v0.h[0]
|
||||
{% if flipped %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} v{{reg}}.8h, v{{reg}}.8h, v0.8h
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} v{{reg}}.8h, v0.8h, v{{reg}}.8h
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x5, [x1, #128]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr x6, [x1, #136]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x7, [x1, #144]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr x9, [x1, #152]
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x10, [x1, #96]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr x11, [x1, #104]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x12, [x1, #112]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr x13, [x1, #120]
|
||||
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [ x1 ]
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
ldr x14, [x1, #128]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
ldr x15, [x1, #136]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
ldr x20, [x1, #144]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
ldr x21, [x1, #152]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
ldr x22, [x1, #160]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
ldr x23, [x1, #168]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
ldr x24, [x1, #176]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
ldr x25, [x1, #184]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v1.d[0], x7
|
||||
ins v2.d[0], x10
|
||||
ins v3.d[0], x12
|
||||
ins v4.d[0], x14
|
||||
ins v5.d[0], x20
|
||||
ins v6.d[0], x22
|
||||
ins v7.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v1.d[1], x9
|
||||
ins v2.d[1], x11
|
||||
ins v3.d[1], x13
|
||||
ins v4.d[1], x15
|
||||
ins v5.d[1], x21
|
||||
ins v6.d[1], x23
|
||||
ins v7.d[1], x25
|
||||
|
||||
add x1, x1, #192
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
ld1 {{ v9.8h, v10.8h, v11.8h, v12.8h }}, [x1], #64
|
||||
ld1 {{ v13.8h, v14.8h, v15.8h }}, [x1], #48
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
fmla v24.8h, v9.8h, v8.h[0]
|
||||
ld1 {{ v9.8h }}, [ x1 ], #16
|
||||
ld1 {{ v0.8h, v1.8h, v2.8h, v3.8h }}, [x1], #64
|
||||
ld1 {{ v4.8h, v5.8h, v6.8h, v7.8h }}, [x1], #64
|
||||
fmla v25.8h, v10.8h, v8.h[0]
|
||||
fmla v26.8h, v11.8h, v8.h[0]
|
||||
fmla v27.8h, v12.8h, v8.h[0]
|
||||
fmla v28.8h, v13.8h, v8.h[0]
|
||||
fmla v29.8h, v14.8h, v8.h[0]
|
||||
fmla v30.8h, v15.8h, v8.h[0]
|
||||
|
||||
fmla v31.8h, v9.8h, v8.h[0]
|
||||
|
||||
ld1 {{ v8.h }}[0], [ x2 ], #2
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s }}, [x1], #48
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ldr w8, [x2], #4
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
ldr d12, [x1], #8
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
ldr x12, [x1], #8
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
ldr d14, [x1], #8
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
ldr x14, [x1], #8
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
ldr d15, [x1], #8
|
||||
fmla v24.8h, v9.8h, v8.h[0]
|
||||
ldr x15, [x1], #8
|
||||
|
||||
ld1 {{ v0.8h, v1.8h, v2.8h, v3.8h }}, [x1], #64
|
||||
ins v8.s[1], w8
|
||||
ld1 {{ v4.8h, v5.8h, v6.8h, v7.8h }}, [x1], #64
|
||||
|
||||
fmla v25.8h, v10.8h, v8.h[0]
|
||||
ins v12.d[1], x12
|
||||
fmla v26.8h, v11.8h, v8.h[0]
|
||||
ins v13.d[1], x13
|
||||
fmla v27.8h, v12.8h, v8.h[0]
|
||||
ins v14.d[1], x14
|
||||
fmla v28.8h, v13.8h, v8.h[0]
|
||||
ins v15.d[1], x15
|
||||
|
||||
ld1 {{ v9.8h, v10.8h, v11.8h, v12.8h }}, [x1], #64
|
||||
|
||||
fmla v29.8h, v14.8h, v8.h[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v30.8h, v15.8h, v8.h[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v31.8h, v0.8h, v8.h[0]
|
||||
ldr d14, [x1], #8
|
||||
|
||||
fmla v16.8h, v1.8h, v8.h[2]
|
||||
ldr x14, [x1], #8
|
||||
fmla v17.8h, v2.8h, v8.h[2]
|
||||
ldr d15, [x1], #8
|
||||
fmla v18.8h, v3.8h, v8.h[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v19.8h, v4.8h, v8.h[2]
|
||||
|
||||
ld1 {{ v0.8h }}, [x1], #16
|
||||
|
||||
fmla v20.8h, v5.8h, v8.h[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.8h, v6.8h, v8.h[2]
|
||||
ldr x10, [x1], #8
|
||||
|
||||
fmla v22.8h, v7.8h, v8.h[2]
|
||||
|
||||
fmla v23.8h, v9.8h, v8.h[2]
|
||||
ins v13.d[1], x13
|
||||
fmla v24.8h, v10.8h, v8.h[2]
|
||||
ins v14.d[1], x14
|
||||
fmla v25.8h, v11.8h, v8.h[2]
|
||||
ins v15.d[1], x15
|
||||
|
||||
fmla v26.8h, v12.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
fmla v27.8h, v13.8h, v8.h[2]
|
||||
ins v1.d[1], x10
|
||||
fmla v28.8h, v14.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
fmla v29.8h, v15.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
fmla v30.8h, v0.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
fmla v31.8h, v1.8h, v8.h[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.h[0], v8.h[3]
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// v8..v15 are callee-preserved
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_128x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_128x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
sub x3, x3, #1
|
||||
|
||||
|
||||
ld1 { v8.h }[0], [ x2 ], #2
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% include "arm64fp16_mmm_f16_128x1/loop1/naive.tmpli" %}
|
||||
{% endcapture %}
|
||||
|
||||
{% capture packed_packed_loop2 %}
|
||||
{% include "arm64fp16_mmm_f16_128x1/loop2/cortex_a55.tmpli" %}
|
||||
{% endcapture %}
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
// last loop can't read beyond actual input as it's likely not packed and padded
|
||||
.packed_packed_loop_1_last:
|
||||
ld1 { v9.8h, v10.8h, v11.8h, v12.8h }, [x1], #64
|
||||
ld1 { v13.8h, v14.8h, v15.8h }, [x1], #48
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
ld1 { v0.8h }, [ x1 ]
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
|
||||
fmla v24.8h, v9.8h, v8.h[0]
|
||||
fmla v25.8h, v10.8h, v8.h[0]
|
||||
fmla v26.8h, v11.8h, v8.h[0]
|
||||
fmla v27.8h, v12.8h, v8.h[0]
|
||||
fmla v28.8h, v13.8h, v8.h[0]
|
||||
fmla v29.8h, v14.8h, v8.h[0]
|
||||
fmla v30.8h, v15.8h, v8.h[0]
|
||||
fmla v31.8h, v0.8h, v8.h[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64fp16_mmm_f16_scalars.tmpliq" from:16, to:31%}
|
||||
{% include "arm64fp16_mmm_f16_per_rows.tmpliq" mr:128, from:16, to:31%}
|
||||
{% include "arm64fp16_mmm_f16_per_cols.tmpliq" mr:128, from:16, to:31%}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #2
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..7) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.8h, v{{reg}}.8h, v0.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.8h-v3.8h}, [x5], #64
|
||||
ld1 {v4.8h-v7.8h}, [x5], #64
|
||||
ld1 {v8.8h-v11.8h}, [x5], #64
|
||||
ld1 {v12.8h-v15.8h}, [x5], #64
|
||||
|
||||
{% for r in (0..15) %}
|
||||
fadd v{{r| plus: 16}}.8h, v{{r | plus: 16}}.8h, v{{r}}.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.h}[0], [ x3 ]
|
||||
|
||||
{% for r in (0..7) %}
|
||||
ldr q{{r}}, [x2], #16
|
||||
{% endfor %}
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ldr q0, [x2], #16
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
ldr q1, [x2], #16
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
ldr q2, [x2], #16
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
ldr q3, [x2], #16
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
ldr q4, [x2], #16
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
ldr q5, [x2], #16
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
ldr q6, [x2], #16
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
ldr q7, [x2], #16
|
||||
|
||||
fmla v24.8h, v0.8h, v8.h[0]
|
||||
fmla v25.8h, v1.8h, v8.h[0]
|
||||
fmla v26.8h, v2.8h, v8.h[0]
|
||||
fmla v27.8h, v3.8h, v8.h[0]
|
||||
fmla v28.8h, v4.8h, v8.h[0]
|
||||
fmla v29.8h, v5.8h, v8.h[0]
|
||||
fmla v30.8h, v6.8h, v8.h[0]
|
||||
fmla v31.8h, v7.8h, v8.h[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc$
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..7) %}
|
||||
st1 { v{{reg}}.h }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
st1 { v{{reg}}.8h }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v0.8h, v4.h[1]
|
||||
fmla v19.8h, v1.8h, v4.h[1]
|
||||
fmla v20.8h, v0.8h, v4.h[2]
|
||||
fmla v21.8h, v1.8h, v4.h[2]
|
||||
fmla v22.8h, v0.8h, v4.h[3]
|
||||
fmla v23.8h, v1.8h, v4.h[3]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[4]
|
||||
fmla v25.8h, v1.8h, v4.h[4]
|
||||
fmla v26.8h, v0.8h, v4.h[5]
|
||||
fmla v27.8h, v1.8h, v4.h[5]
|
||||
fmla v28.8h, v0.8h, v4.h[6]
|
||||
fmla v29.8h, v1.8h, v4.h[6]
|
||||
fmla v30.8h, v0.8h, v4.h[7]
|
||||
fmla v31.8h, v1.8h, v4.h[7]
|
||||
|
||||
ld1 {{ v0.8h, v1.8h }}, [x1], #32
|
||||
ld1 {{ v4.8h }}, [x2], #16
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
ldr d2, [x1], #8
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
ldr d6, [x2], #8
|
||||
fmla v18.8h, v0.8h, v4.h[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v1.8h, v4.h[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.8h, v0.8h, v4.h[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v21.8h, v1.8h, v4.h[2]
|
||||
fmla v22.8h, v0.8h, v4.h[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.8h, v1.8h, v4.h[3]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[4]
|
||||
fmla v25.8h, v1.8h, v4.h[4]
|
||||
fmla v26.8h, v0.8h, v4.h[5]
|
||||
fmla v27.8h, v1.8h, v4.h[5]
|
||||
fmla v28.8h, v0.8h, v4.h[6]
|
||||
ins v2.d[1], x5
|
||||
fmla v29.8h, v1.8h, v4.h[6]
|
||||
ins v6.d[1], x7
|
||||
fmla v30.8h, v0.8h, v4.h[7]
|
||||
ins v3.d[1], x6
|
||||
fmla v31.8h, v1.8h, v4.h[7]
|
||||
|
||||
fmla v16.8h, v2.8h, v6.h[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.8h, v3.8h, v6.h[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.8h, v2.8h, v6.h[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v3.8h, v6.h[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.8h, v2.8h, v6.h[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.8h, v3.8h, v6.h[2]
|
||||
fmla v22.8h, v2.8h, v6.h[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.8h, v3.8h, v6.h[3]
|
||||
|
||||
fmla v24.8h, v2.8h, v6.h[4]
|
||||
fmla v25.8h, v3.8h, v6.h[4]
|
||||
fmla v26.8h, v2.8h, v6.h[5]
|
||||
fmla v27.8h, v3.8h, v6.h[5]
|
||||
fmla v28.8h, v2.8h, v6.h[6]
|
||||
ins v0.d[1], x5
|
||||
fmla v29.8h, v3.8h, v6.h[6]
|
||||
ins v4.d[1], x7
|
||||
fmla v30.8h, v2.8h, v6.h[7]
|
||||
ins v1.d[1], x6
|
||||
fmla v31.8h, v3.8h, v6.h[7]
|
||||
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// x20..x27 are used, callee-preserved
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_16x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_16x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
str q8, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed:
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s }, [ x2 ], #16
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% include "arm64fp16_mmm_f16_16x8/loop1/naive.tmpli" %}
|
||||
{% endcapture %}
|
||||
|
||||
{% capture packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64fp16_mmm_f16_16x8/loop2/cortex_a55.tmpli" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64fp16_mmm_f16_scalars.tmpliq" from:16, to:31%}
|
||||
{% include "arm64fp16_mmm_f16_per_rows.tmpliq" mr:16, from:16, to:31 %}
|
||||
{% include "arm64fp16_mmm_f16_per_cols.tmpliq" mr:16, from:16, to:31 %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..7) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{col | times:2 | plus: reg}}.8h, v{{col | times:2 | plus: reg}}.8h, v0.8h
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ], #32
|
||||
ld1 { v4.4s }, [ x3 ], #16
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v0.8h, v4.h[1]
|
||||
fmla v19.8h, v1.8h, v4.h[1]
|
||||
fmla v20.8h, v0.8h, v4.h[2]
|
||||
fmla v21.8h, v1.8h, v4.h[2]
|
||||
fmla v22.8h, v0.8h, v4.h[3]
|
||||
fmla v23.8h, v1.8h, v4.h[3]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[4]
|
||||
fmla v25.8h, v1.8h, v4.h[4]
|
||||
fmla v26.8h, v0.8h, v4.h[5]
|
||||
fmla v27.8h, v1.8h, v4.h[5]
|
||||
fmla v28.8h, v0.8h, v4.h[6]
|
||||
fmla v29.8h, v1.8h, v4.h[6]
|
||||
fmla v30.8h, v0.8h, v4.h[7]
|
||||
fmla v31.8h, v1.8h, v4.h[7]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #2
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in (8..15) %}
|
||||
str q{{col | times:2 }}, [ x5 ]
|
||||
str q{{col | times:2 | plus: 1}}, [ x5, #16 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..7) %}
|
||||
st1 { v{{col | times:2 | plus: reg}}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldr q8, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v2.8h, v4.h[0]
|
||||
fmla v19.8h, v3.8h, v4.h[0]
|
||||
fmla v20.8h, v0.8h, v4.h[1]
|
||||
fmla v21.8h, v1.8h, v4.h[1]
|
||||
fmla v22.8h, v2.8h, v4.h[1]
|
||||
fmla v23.8h, v3.8h, v4.h[1]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[2]
|
||||
fmla v25.8h, v1.8h, v4.h[2]
|
||||
fmla v26.8h, v2.8h, v4.h[2]
|
||||
fmla v27.8h, v3.8h, v4.h[2]
|
||||
fmla v28.8h, v0.8h, v4.h[3]
|
||||
fmla v29.8h, v1.8h, v4.h[3]
|
||||
fmla v30.8h, v2.8h, v4.h[3]
|
||||
fmla v31.8h, v3.8h, v4.h[3]
|
||||
|
||||
ld1 {{ v0.8h, v1.8h, v2.8h, v3.8h }}, [ x1 ], #64
|
||||
ldr d4, [x2], #8
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
// mul a: v0, v1, v2, v3 b: v4
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9 as d9
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.8h, v2.8h, v4.h[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v3.8h, v4.h[0]
|
||||
fmla v20.8h, v0.8h, v4.h[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.8h, v1.8h, v4.h[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.8h, v2.8h, v4.h[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.8h, v3.8h, v4.h[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.8h, v1.8h, v4.h[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.8h, v2.8h, v4.h[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.8h, v3.8h, v4.h[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.8h, v0.8h, v4.h[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.8h, v1.8h, v4.h[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.8h, v2.8h, v4.h[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.8h, v3.8h, v4.h[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4 as d4
|
||||
|
||||
fmla v16.8h, v5.8h, v9.h[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.8h, v6.8h, v9.h[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.8h, v7.8h, v9.h[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v8.8h, v9.h[0]
|
||||
fmla v20.8h, v5.8h, v9.h[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.8h, v6.8h, v9.h[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.8h, v7.8h, v9.h[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.8h, v8.8h, v9.h[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.8h, v5.8h, v9.h[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.8h, v6.8h, v9.h[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.8h, v7.8h, v9.h[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.8h, v8.8h, v9.h[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.8h, v5.8h, v9.h[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.8h, v6.8h, v9.h[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.8h, v7.8h, v9.h[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.8h, v8.8h, v9.h[3]
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// x20..x27 are used, callee-preserved
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_32x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_32x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ldr d4, [x2], #8
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% include "arm64fp16_mmm_f16_32x4/loop1/naive.tmpli" %}
|
||||
{% endcapture %}
|
||||
|
||||
{% capture packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64fp16_mmm_f16_32x4/loop2/cortex_a55.tmpli" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64fp16_mmm_f16_scalars.tmpliq" from:16, to:31%}
|
||||
{% include "arm64fp16_mmm_f16_per_rows.tmpliq" mr:32, from:16, to:31 %}
|
||||
{% include "arm64fp16_mmm_f16_per_cols.tmpliq" mr:32, from:16, to:31 %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..3) %}
|
||||
{% for lane in (0..7) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{col | times:4 | plus: 16| plus: reg}}.8h, v{{col | times:4 | plus: 16 | plus: reg}}.8h, v0.8h
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x2 ]
|
||||
ldr d4, [x3]
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v2.8h, v4.h[0]
|
||||
fmla v19.8h, v3.8h, v4.h[0]
|
||||
fmla v20.8h, v0.8h, v4.h[1]
|
||||
fmla v21.8h, v1.8h, v4.h[1]
|
||||
fmla v22.8h, v2.8h, v4.h[1]
|
||||
fmla v23.8h, v3.8h, v4.h[1]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[2]
|
||||
fmla v25.8h, v1.8h, v4.h[2]
|
||||
fmla v26.8h, v2.8h, v4.h[2]
|
||||
fmla v27.8h, v3.8h, v4.h[2]
|
||||
fmla v28.8h, v0.8h, v4.h[3]
|
||||
fmla v29.8h, v1.8h, v4.h[3]
|
||||
fmla v30.8h, v2.8h, v4.h[3]
|
||||
fmla v31.8h, v3.8h, v4.h[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #2
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in (0..3) %}
|
||||
str q{{col | times:4 | plus:16 | plus: 0}}, [ x5 ]
|
||||
str q{{col | times:4 | plus:16 | plus: 1}}, [ x5, #16 ]
|
||||
str q{{col | times:4 | plus:16 | plus: 2}}, [ x5, #32 ]
|
||||
str q{{col | times:4 | plus:16 | plus: 3}}, [ x5, #48 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..3) %}
|
||||
{% for lane in (0..7) %}
|
||||
st1 { v{{col | times:4 | plus: 16 | plus: reg}}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_min", op:"fmin", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_max", op:"fmax", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_mul", op:"fmul", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_add", op:"fadd", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_sub", op:"fsub", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_col.tmpliq" label:"per_col_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_min", op:"fmin", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_max", op:"fmax", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_mul", op:"fmul", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_add", op:"fadd", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_sub", op:"fsub", mr:mr, from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_per_row.tmpliq" label:"per_row_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64fp16_mmm_8h_scalar.tmpliq" label:"scalar_min", op:"fmin", from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_scalar.tmpliq" label:"scalar_max", op:"fmax", from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_scalar.tmpliq" label:"scalar_mul", op:"fmul", from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_scalar.tmpliq" label:"scalar_add", op:"fadd", from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_scalar.tmpliq" label:"scalar_sub", op:"fsub", from:from, to:to %}
|
||||
{% include "arm64fp16_mmm_8h_scalar.tmpliq" label:"scalar_sub_flipped", op:"fsub", from:from, to:to, flipped:true %}
|
||||
|
||||
.clear:
|
||||
{% for r in (from..to) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_sigmoid_f16_8n_{{suffix}}
|
||||
{{G}}arm64fp16_sigmoid_f16_8n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.8h }, [x2]
|
||||
dup v5.8h, v0.h[0] // v5 <- low, broadcasted
|
||||
dup v6.8h, v0.h[1] // v6 <- high, broadcasted
|
||||
dup v7.8h, v0.h[7] // v7 <- half, broadcasted
|
||||
|
||||
cmp x1, #32
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmax v17.8h, v17.8h, v5.8h
|
||||
fmax v18.8h, v18.8h, v5.8h
|
||||
fmax v19.8h, v19.8h, v5.8h
|
||||
|
||||
fmin v16.8h, v16.8h, v6.8h
|
||||
fmin v17.8h, v17.8h, v6.8h
|
||||
fmin v18.8h, v18.8h, v6.8h
|
||||
fmin v19.8h, v19.8h, v6.8h // v16 <- x
|
||||
|
||||
fmul v20.8h, v16.8h, v16.8h
|
||||
fmul v21.8h, v17.8h, v17.8h
|
||||
fmul v22.8h, v18.8h, v18.8h
|
||||
fmul v23.8h, v19.8h, v19.8h // v20 <- x2
|
||||
|
||||
dup v28.8h, v0.h[3]
|
||||
fmla v28.8h, v20.8h, v0.h[2]
|
||||
dup v29.8h, v0.h[3]
|
||||
fmla v29.8h, v21.8h, v0.h[2]
|
||||
dup v30.8h, v0.h[3]
|
||||
fmla v30.8h, v22.8h, v0.h[2]
|
||||
dup v31.8h, v0.h[3]
|
||||
fmla v31.8h, v23.8h, v0.h[2]
|
||||
|
||||
dup v24.8h, v0.h[4]
|
||||
fmla v24.8h, v20.8h, v28.8h
|
||||
dup v25.8h, v0.h[4]
|
||||
fmla v25.8h, v21.8h, v29.8h
|
||||
dup v26.8h, v0.h[4]
|
||||
fmla v26.8h, v22.8h, v30.8h
|
||||
dup v27.8h, v0.h[4]
|
||||
fmla v27.8h, v23.8h, v31.8h
|
||||
|
||||
fmul v16.8h, v16.8h, v24.8h
|
||||
fmul v17.8h, v17.8h, v25.8h
|
||||
fmul v18.8h, v18.8h, v26.8h
|
||||
fmul v19.8h, v19.8h, v27.8h // v16 <- numerator
|
||||
|
||||
dup v24.8h, v0.h[6]
|
||||
dup v25.8h, v0.h[6]
|
||||
dup v26.8h, v0.h[6]
|
||||
dup v27.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v0.h[5]
|
||||
fmla v25.8h, v21.8h, v0.h[5]
|
||||
fmla v26.8h, v22.8h, v0.h[5]
|
||||
fmla v27.8h, v23.8h, v0.h[5] // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
fdiv v17.8h, v17.8h, v25.8h
|
||||
fdiv v18.8h, v18.8h, v26.8h
|
||||
fdiv v19.8h, v19.8h, v27.8h
|
||||
|
||||
fadd v16.8h, v16.8h, v7.8h
|
||||
fadd v17.8h, v17.8h, v7.8h
|
||||
fadd v18.8h, v18.8h, v7.8h
|
||||
fadd v19.8h, v19.8h, v7.8h
|
||||
|
||||
st1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0], #64
|
||||
|
||||
subs x1, x1, #32
|
||||
cmp x1, #32
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmin v16.8h, v16.8h, v6.8h // v16 <- x
|
||||
fmul v20.8h, v16.8h, v16.8h // v20 <- x2
|
||||
|
||||
dup v28.8h, v0.h[3]
|
||||
fmla v28.8h, v20.8h, v0.h[2]
|
||||
dup v24.8h, v0.h[4]
|
||||
fmla v24.8h, v20.8h, v28.8h
|
||||
fmul v16.8h, v16.8h, v24.8h // v16 <- numerator
|
||||
|
||||
dup v24.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v0.h[5] // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
fadd v16.8h, v16.8h, v7.8h
|
||||
|
||||
st1 { v16.8h }, [x0], #16
|
||||
|
||||
subs x1, x1, #8
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
{{ -6.92 | float16 }}
|
||||
{{ 6.92 | float16 }}
|
||||
{{ -0.0000124702 | float16 }}
|
||||
{{ 0.00400222 | float16 }}
|
||||
|
||||
{{ 0.249895 | float16 }}
|
||||
{{ 0.098734 | float16 }}
|
||||
{{ 1.0 | float16 }}
|
||||
{{ 0.5 | float16 }}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_tanh_f16_8n_{{suffix}}
|
||||
{{G}}arm64fp16_tanh_f16_8n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.8h }, [x2]
|
||||
dup v5.8h, v0.h[0] // v5 <- low, broadcasted
|
||||
dup v6.8h, v0.h[1] // v6 <- high, broadcasted
|
||||
|
||||
cmp x1, #32
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmax v17.8h, v17.8h, v5.8h
|
||||
fmax v18.8h, v18.8h, v5.8h
|
||||
fmax v19.8h, v19.8h, v5.8h
|
||||
|
||||
fmin v16.8h, v16.8h, v6.8h
|
||||
fmin v17.8h, v17.8h, v6.8h
|
||||
fmin v18.8h, v18.8h, v6.8h
|
||||
fmin v19.8h, v19.8h, v6.8h // v16 <- x
|
||||
|
||||
fmul v20.8h, v16.8h, v16.8h
|
||||
fmul v21.8h, v17.8h, v17.8h
|
||||
fmul v22.8h, v18.8h, v18.8h
|
||||
fmul v23.8h, v19.8h, v19.8h // v20 <- x2
|
||||
|
||||
dup v24.8h, v0.h[3]
|
||||
fmla v24.8h, v20.8h, v0.h[2]
|
||||
dup v25.8h, v0.h[3]
|
||||
fmla v25.8h, v21.8h, v0.h[2]
|
||||
dup v26.8h, v0.h[3]
|
||||
fmla v26.8h, v22.8h, v0.h[2]
|
||||
dup v27.8h, v0.h[3]
|
||||
fmla v27.8h, v23.8h, v0.h[2]
|
||||
|
||||
fmul v16.8h, v16.8h, v24.8h
|
||||
fmul v17.8h, v17.8h, v25.8h
|
||||
fmul v18.8h, v18.8h, v26.8h
|
||||
fmul v19.8h, v19.8h, v27.8h // v16 <- numerator
|
||||
|
||||
dup v28.8h, v0.h[5]
|
||||
fmla v28.8h, v20.8h, v0.h[4]
|
||||
dup v29.8h, v0.h[5]
|
||||
fmla v29.8h, v21.8h, v0.h[4]
|
||||
dup v30.8h, v0.h[5]
|
||||
fmla v30.8h, v22.8h, v0.h[4]
|
||||
dup v31.8h, v0.h[5]
|
||||
fmla v31.8h, v23.8h, v0.h[4]
|
||||
|
||||
dup v24.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v28.8h
|
||||
dup v25.8h, v0.h[6]
|
||||
fmla v25.8h, v21.8h, v29.8h
|
||||
dup v26.8h, v0.h[6]
|
||||
fmla v26.8h, v22.8h, v30.8h
|
||||
dup v27.8h, v0.h[6]
|
||||
fmla v27.8h, v23.8h, v31.8h // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
fdiv v17.8h, v17.8h, v25.8h
|
||||
fdiv v18.8h, v18.8h, v26.8h
|
||||
fdiv v19.8h, v19.8h, v27.8h
|
||||
|
||||
st1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0], #64
|
||||
|
||||
subs x1, x1, #32
|
||||
cmp x1, #32
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmin v16.8h, v16.8h, v6.8h // v16 <- x
|
||||
fmul v20.8h, v16.8h, v16.8h // v20 <- x2
|
||||
|
||||
dup v24.8h, v0.h[3]
|
||||
fmla v24.8h, v20.8h, v0.h[2]
|
||||
fmul v16.8h, v16.8h, v24.8h // v16 <- numerator
|
||||
|
||||
dup v28.8h, v0.h[5]
|
||||
fmla v28.8h, v20.8h, v0.h[4]
|
||||
dup v24.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v28.8h // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
|
||||
st1 { v16.8h }, [x0], #16
|
||||
|
||||
subs x1, x1, #8
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
{{ -3.84 | float16 }}
|
||||
{{ 3.84 | float16 }}
|
||||
{{ 0.082654955 | float16 }} // alpha
|
||||
{{ 0.99963124 | float16 }}
|
||||
|
||||
{{ 0.0065383179 | float16 }} // beta
|
||||
{{ 0.41401828 | float16 }}
|
||||
{{ 1.0 | float16 }}
|
||||
{{ 0 | float16 }} // padding
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
sub x0, x0, 40
|
||||
|
||||
.non_linear_loop:
|
||||
add x0, x0, 40
|
||||
ldr x2, [x0]
|
||||
|
||||
mov x4, #{{ jump_table | size }}
|
||||
|
||||
cmp x2, #{{ jump_table | size }}
|
||||
csel x2, x2, x4, lt
|
||||
cmp x2, #0
|
||||
csel x2, x4, x2, lt
|
||||
|
||||
adr x3, .jmp_table
|
||||
add x3, x3, x2, LSL#2
|
||||
br x3
|
||||
|
||||
.jmp_table:
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
add x0, x2, #4000
|
||||
b .return
|
||||
|
||||
.unsupported:
|
||||
mov x0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov x0, 0
|
||||
b .return
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// serves as a canary build file to figure out which flag combination will accept half precision fmla
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
// .cpu generic+fp+simd+fp16
|
||||
.global foo
|
||||
foo:
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ret
|
||||
|
||||
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// serves as a canary build file to figure out which flag combination will accept half precision fmla
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd+fp16
|
||||
.global foo
|
||||
foo:
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ret
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% capture mr_over_4 %}{{ mr | divided_by: 4}}{%endcapture%}
|
||||
{% capture cols%}{{to | plus: 1| minus:from| divided_by:mr_over_4}}{%endcapture%}
|
||||
|
||||
{% capture loads %}{{cols | divided_by:4}}{% endcapture %}
|
||||
|
||||
{%if cols == "1" %}
|
||||
ld1 {v0.s}[0], [ x2 ]
|
||||
{% else %}
|
||||
{% for reg in (1..loads) %}
|
||||
ldr q{{reg |minus:1}}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
// {{mr}} {{cols}}
|
||||
|
||||
{% for col in (1..cols) %}
|
||||
dup v3.4s, v{{col| minus: 1|divided_by:4}}.s[{{col| minus: 1|modulo:4}}]
|
||||
{% for row in (1..mr_over_4) %}
|
||||
{% capture acc %}{{ col|minus:1|times:mr_over_4|plus:row|minus:1|plus:from }}{% endcapture %}
|
||||
{% if flipped %}
|
||||
{{op}} v{{acc}}.4s, v{{acc}}.4s, v3.4s
|
||||
{% else %}
|
||||
{{op}} v{{acc}}.4s, v3.4s, v{{acc}}.4s
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% capture mr_over_4 %}{{ mr | divided_by: 4}}{%endcapture%}
|
||||
{% capture mr_over_4_min_1 %}{{ mr | divided_by: 4 | minus: 1}}{%endcapture%}
|
||||
|
||||
{% for reg in (0..mr_over_4_min_1) %}
|
||||
ldr q{{reg}}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%}
|
||||
{{op}} v{{acc}}.4s, v{{acc}}.4s, v{{other}}.4s
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in (from..to) %}
|
||||
{% capture other%}{{acc | minus: from | modulo: mr_over_4}}{%endcapture%}
|
||||
{{op}} v{{acc}}.4s, v{{other}}.4s, v{{acc}}.4s
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.{{label}}:
|
||||
add x2, x0, #8
|
||||
ld1 {v0.s}[0], [ x2 ]
|
||||
dup v0.4s, v0.s[0]
|
||||
{% if flipped %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in (from..to) %}
|
||||
{{op}} v{{reg}}.4s, v0.4s, v{{reg}}.4s
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr w4, [x1], #4
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr w22, [x2], #4
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr w23, [x2], #4
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr w8, [x1], #4
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr w24, [x2], #4
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr w9, [x1], #4
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
ldr w25, [x2], #4
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
ldr w10, [x1], #4
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ldr w26, [x2], #4
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
ldr w11, [x1], #4
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
ldr w27, [x2], #4
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
ldr w12, [x1], #4
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
ldr w13, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
ldr w14, [x1], #4
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
ldr w15, [x1], #4
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ins v0.s[0], w4
|
||||
ins v1.s[0], w8
|
||||
ins v2.s[0], w12
|
||||
ins v4.s[0], w20
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[1], w5
|
||||
ins v1.s[1], w9
|
||||
ins v2.s[1], w13
|
||||
ins v4.s[1], w21
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[2], w6
|
||||
ins v1.s[2], w10
|
||||
ins v2.s[2], w14
|
||||
ins v4.s[2], w22
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[3], w7
|
||||
ins v1.s[3], w11
|
||||
ins v2.s[3], w15
|
||||
ins v4.s[3], w23
|
||||
ins v5.s[3], w27
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr w4, [x1]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr w5, [x1, #4]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr w6, [x1, #8]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr w22, [x2], #4
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1, #12]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr w23, [x2], #4
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr w8, [x1, #16]
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr w24, [x2], #4
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr w9, [x1, #20]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
ldr w25, [x2], #4
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
ldr w10, [x1, #24]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ldr w26, [x2], #4
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
ldr w11, [x1, #28]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
ldr w27, [x2], #4
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
ldr w12, [x1, #32]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
ldr w13, [x1, #36]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
ldr w14, [x1, #40]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
ldr w15, [x1, #44]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
prfm pldl1keep, [x1, #512]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
add x1, x1, #48
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ins v0.s[0], w4
|
||||
|
||||
ins v1.s[0], w8
|
||||
ins v2.s[0], w12
|
||||
|
||||
ins v4.s[0], w20
|
||||
ins v5.s[0], w24
|
||||
|
||||
ins v0.s[1], w5
|
||||
ins v1.s[1], w9
|
||||
|
||||
ins v2.s[1], w13
|
||||
ins v4.s[1], w21
|
||||
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[2], w6
|
||||
|
||||
ins v1.s[2], w10
|
||||
ins v2.s[2], w14
|
||||
|
||||
ins v4.s[2], w22
|
||||
ins v5.s[2], w26
|
||||
|
||||
ins v0.s[3], w7
|
||||
ins v1.s[3], w11
|
||||
|
||||
ins v2.s[3], w15
|
||||
ins v4.s[3], w23
|
||||
ins v5.s[3], w27
|
||||
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr x4, [x1]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr x20, [x2]
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1, #8]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr x21, [x2, #8]
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1, #16]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr x22, [x2, #16]
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1, #24]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr x23, [x2, #24]
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr x8, [x1, #32]
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr x9, [x1, #40]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
prfm pldl1keep, [x1, #512]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
add x1, x1, #48
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
add x2, x2, #32
|
||||
|
||||
|
||||
ins v0.d[0], x4
|
||||
ins v2.d[0], x8
|
||||
|
||||
ins v4.d[0], x20
|
||||
ins v5.d[0], x22
|
||||
|
||||
ins v0.d[1], x5
|
||||
ins v2.d[1], x9
|
||||
|
||||
ins v4.d[1], x21
|
||||
ins v1.d[0], x6
|
||||
|
||||
ins v1.d[1], x7
|
||||
|
||||
ins v5.d[1], x23
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s }}, [x1], #48
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
// mul a: v0, v1, v2, b: v4, v5
|
||||
// load a: d3/x23, d6/x26, d7/x27
|
||||
// load b: x4, x5, x6, x7
|
||||
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr d3, [x1], #8
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr x4, [x2], #8
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr x23, [x1], #8
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr x5, [x2], #8
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr x6, [x2], #8
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr x26, [x1], #8
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr x7, [x2], #8
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr d7, [x1], #8
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr x27, [x1], #8
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
|
||||
// ins b: v4 <- x4/x5
|
||||
// ins a: d3/x23, d6/x26, d7/x27
|
||||
|
||||
ins v4.d[0], x4
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ins v4.d[1], x5
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
ins v3.d[1], x23
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
ins v6.d[1], x26
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
ins v7.d[1], x27
|
||||
|
||||
// mul a: v3, v6, v7, b: v4, v5
|
||||
// ins b, v5 <- x6, x7
|
||||
// load a: d0/x20, d1/x21, d2/x22
|
||||
// load b: x4, x5
|
||||
|
||||
fmla v8.4s, v3.4s, v4.s[0]
|
||||
ins v5.d[0], x6
|
||||
fmla v9.4s, v6.4s, v4.s[0]
|
||||
ins v5.d[1], x7
|
||||
fmla v10.4s, v7.4s, v4.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v11.4s, v3.4s, v4.s[1]
|
||||
ldr x4, [x2], #8
|
||||
fmla v12.4s, v6.4s, v4.s[1]
|
||||
ldr x20, [x1], #8
|
||||
fmla v13.4s, v7.4s, v4.s[1]
|
||||
ldr x5, [x2], #8
|
||||
fmla v14.4s, v3.4s, v4.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v15.4s, v6.4s, v4.s[2]
|
||||
ldr x6, [x2], #8
|
||||
fmla v16.4s, v7.4s, v4.s[2]
|
||||
ldr x21, [x1], #8
|
||||
fmla v17.4s, v3.4s, v4.s[3]
|
||||
ldr x7, [x2], #8
|
||||
|
||||
// load b: x6, x7
|
||||
fmla v18.4s, v6.4s, v4.s[3]
|
||||
ldr d2, [x1], #8
|
||||
fmla v19.4s, v7.4s, v4.s[3]
|
||||
ldr x22, [x1], #8
|
||||
fmla v20.4s, v3.4s, v5.s[0]
|
||||
fmla v21.4s, v6.4s, v5.s[0]
|
||||
fmla v22.4s, v7.4s, v5.s[0]
|
||||
fmla v23.4s, v3.4s, v5.s[1]
|
||||
fmla v24.4s, v6.4s, v5.s[1]
|
||||
fmla v25.4s, v7.4s, v5.s[1]
|
||||
|
||||
// ins a: d0/x20, d1/x21, d2/x22
|
||||
fmla v26.4s, v3.4s, v5.s[2]
|
||||
ins v0.d[1], x20
|
||||
fmla v27.4s, v6.4s, v5.s[2]
|
||||
ins v1.d[1], x21
|
||||
fmla v28.4s, v7.4s, v5.s[2]
|
||||
ins v2.d[1], x22
|
||||
|
||||
// ins b: v4 <- x4, x5
|
||||
fmla v29.4s, v3.4s, v5.s[3]
|
||||
ins v4.d[0], x4
|
||||
fmla v30.4s, v6.4s, v5.s[3]
|
||||
ins v4.d[1], x5
|
||||
fmla v31.4s, v7.4s, v5.s[3]
|
||||
|
||||
// ins b: v5 <- x6, x7
|
||||
ins v5.d[0], x6
|
||||
ins v5.d[1], x7
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v8 v11 v14 v17 v20 v23 v26 v29
|
||||
// v9 v12 v15 v18 v21 v24 v27 v30
|
||||
// v10 v13 v16 v19 v22 v25 v28 v31
|
||||
|
||||
// no preservation for v0-v7:
|
||||
// packed A buffering (2x8 values): rotating over v0..v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_12x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_12x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s }, [ x1 ], #48
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop1/ldr_x_preload.tmpli" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop1/naive.tmpli" %}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
{% capture packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop2/cortex_a55.tmpli" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_f32_scalars.tmpliq" from:8, to:31%}
|
||||
{% include "arm64simd_mmm_f32_per_rows.tmpliq" mr:12, from:8, to:31 %}
|
||||
{% include "arm64simd_mmm_f32_per_cols.tmpliq" mr:12, from:8, to:31 %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8 ] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
{% for col in (0..7) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..2) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{col | times:3 | plus: 8| plus: reg}}.4s, v{{col | times:3 | plus: 8 | plus: reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
{% for col in (0..7) %}
|
||||
{% for reg in (0..2) %}
|
||||
fmla v{{col | times:3 | plus: 8 | plus: reg}}.4s, v{{reg}}.4s, v{{col| divided_by:4 | plus: 4}}.s[{{col| modulo: 4}}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in (0..7) %}
|
||||
str q{{col | times:3 | plus: 8 }}, [ x5 ]
|
||||
str q{{col | times:3 | plus: 9}}, [ x5, #16 ]
|
||||
str q{{col | times:3 | plus: 10}}, [ x5, #32 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
{% for col in (0..7) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..2) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{col | times:3 | plus: 8 | plus: reg}}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x6, [x1, #8]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x7, [x1, #16]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x8, [x1, #24]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr x9, [x1, #32]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x1, #40]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr x11, [x1, #48]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x12, [x1, #56]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr x24, [x2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x25, [x2, #8]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
add x1, x1, #64
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
add x2, x2, #16
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v2.d[0], x9
|
||||
ins v1.d[0], x7
|
||||
ins v3.d[0], x11
|
||||
ins v4.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v2.d[1], x10
|
||||
ins v1.d[1], x8
|
||||
ins v3.d[1], x12
|
||||
ins v4.d[1], x25
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
ld1 {{ v4.4s }}, [ x2 ], #16
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
// mul a: v0, v1, v2, v3 b: v4
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9(d9/x9)
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4(d4/x9)
|
||||
|
||||
fmla v16.4s, v5.4s, v9.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v6.4s, v9.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v7.4s, v9.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v8.4s, v9.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v5.4s, v9.s[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v9.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v7.4s, v9.s[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.4s, v8.4s, v9.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v5.4s, v9.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.4s, v6.4s, v9.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v7.4s, v9.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.4s, v8.4s, v9.s[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.4s, v5.4s, v9.s[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.4s, v6.4s, v9.s[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.4s, v7.4s, v9.s[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.4s, v8.4s, v9.s[3]
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_16x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_16x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s }, [ x2 ], #16
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop1/cortex_a53.tmpli" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop1/naive.tmpli" %}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
{% capture packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop2/cortex_a55.tmpli" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_f32_scalars.tmpliq" from:16, to:31%}
|
||||
{% include "arm64simd_mmm_f32_per_rows.tmpliq" mr:16, from:16, to:31 %}
|
||||
{% include "arm64simd_mmm_f32_per_cols.tmpliq" mr:16, from:16, to:31 %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..3) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{col | times:4 | plus: 16| plus: reg}}.4s, v{{col | times:4 | plus: 16 | plus: reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ]
|
||||
ld1 { v4.4s }, [ x3 ]
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in (0..3) %}
|
||||
str q{{col | times:4 | plus:16 | plus: 0}}, [ x5 ]
|
||||
str q{{col | times:4 | plus:16 | plus: 1}}, [ x5, #16 ]
|
||||
str q{{col | times:4 | plus:16 | plus: 2}}, [ x5, #32 ]
|
||||
str q{{col | times:4 | plus:16 | plus: 3}}, [ x5, #48 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..3) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{col | times:4 | plus: 16 | plus: reg}}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
// mul a: v0, v1, v2, v3, v4, v5 b: v7
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9(d9/x9)
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4(d4/x9)
|
||||
|
||||
fmla v16.4s, v5.4s, v9.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v6.4s, v9.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v7.4s, v9.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v8.4s, v9.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v5.4s, v9.s[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v9.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v7.4s, v9.s[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.4s, v8.4s, v9.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v5.4s, v9.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.4s, v6.4s, v9.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v7.4s, v9.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.4s, v8.4s, v9.s[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.4s, v5.4s, v9.s[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.4s, v6.4s, v9.s[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.4s, v7.4s, v9.s[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.4s, v8.4s, v9.s[3]
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
ldr x4, [x1]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
ldr x5, [x1, #8]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
ldr x6, [x1, #16]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
ldr x7, [x1, #24]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
ldr x8, [x1, #32]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
ldr x9, [x1, #40]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
ldr x10, [x1, #48]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
ldr x11, [x1, #56]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
ldr x12, [x1, #64]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
ldr x13, [x1, #72]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
ldr x14, [x1, #80]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
ldr x15, [x1, #88]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
ldr x20, [x2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
ldr x21, [x2, #8]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
add x1, x1, #96
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
add x2, x2, #16
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
ins v0.d[0], x4
|
||||
ins v1.d[0], x6
|
||||
ins v2.d[0], x8
|
||||
ins v3.d[0], x10
|
||||
ins v4.d[0], x12
|
||||
ins v5.d[0], x14
|
||||
ins v7.d[0], x20
|
||||
|
||||
ins v0.d[1], x5
|
||||
ins v1.d[1], x7
|
||||
ins v2.d[1], x9
|
||||
ins v3.d[1], x11
|
||||
ins v4.d[1], x13
|
||||
ins v5.d[1], x15
|
||||
ins v7.d[1], x21
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
ldr d0, [x1], #8
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
ldr x20, [x2], #8
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
ldr x21, [x2], #8
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
ldr d1, [x1], #8
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
ldr x7, [x1], #8
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
ldr d2, [x1], #8
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
ldr x9, [x1], #8
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
ins v0.d[1], x5
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
ins v1.d[1], x7
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
ldr d3, [x1], #8
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
ldr x11, [x1], #8
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
ins v2.d[1], x9
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
ldr d4, [x1], #8
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
ldr x14, [x1], #8
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
ins v7.d[0], x20
|
||||
ins v7.d[1], x21
|
||||
|
||||
ins v5.d[0], x14
|
||||
ins v5.d[1], x15
|
||||
|
||||
ins v3.d[1], x11
|
||||
ins v4.d[1], x13
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
ld1 {{ v4.4s, v5.4s }}, [ x1 ], #32
|
||||
ld1 {{ v7.4s }}, [ x2 ], #16
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// x20..x27 are used, callee-preserved
|
||||
|
||||
// C tile regs: v8 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_24x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_24x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s }, [ x1 ], #32
|
||||
ld1 { v7.4s }, [ x2 ], #16
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/cortex_a53.tmpli" %}
|
||||
{% elsif core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/cortex_a55.tmpli" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/naive.tmpli" %}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_f32_scalars.tmpliq" from:8, to:31 %}
|
||||
{% include "arm64simd_mmm_f32_per_rows.tmpliq" mr:24, from:8, to:31 %}
|
||||
{% include "arm64simd_mmm_f32_per_cols.tmpliq" mr:24, from:8, to:31 %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..5) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{col | times:6 | plus: 8 | plus: reg}}.4s, v{{col | times:6 | plus: 8 | plus: reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ], #64
|
||||
ld1 { v7.4s }, [ x3 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ]
|
||||
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in (0..3) %}
|
||||
str q{{col | times:6 | plus:8 | plus: 0}}, [ x5 ]
|
||||
str q{{col | times:6 | plus:8 | plus: 1}}, [ x5, #16 ]
|
||||
str q{{col | times:6 | plus:8 | plus: 2}}, [ x5, #32 ]
|
||||
str q{{col | times:6 | plus:8 | plus: 3}}, [ x5, #48 ]
|
||||
str q{{col | times:6 | plus:8 | plus: 4}}, [ x5, #64 ]
|
||||
str q{{col | times:6 | plus:8 | plus: 5}}, [ x5, #80 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in (0..3) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..5) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{col | times:6 | plus:8 | plus: reg}}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x5, [x1, #128]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr x6, [x1, #136]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x7, [x1, #144]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr x9, [x1, #152]
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x10, [x1, #96]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr x11, [x1, #104]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x12, [x1, #112]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr x13, [x1, #120]
|
||||
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [ x1 ]
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
ldr x14, [x1, #128]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
ldr x15, [x1, #136]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
ldr x20, [x1, #144]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
ldr x21, [x1, #152]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
ldr x22, [x1, #160]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
ldr x23, [x1, #168]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
ldr x24, [x1, #176]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
ldr x25, [x1, #184]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v1.d[0], x7
|
||||
ins v2.d[0], x10
|
||||
ins v3.d[0], x12
|
||||
ins v4.d[0], x14
|
||||
ins v5.d[0], x20
|
||||
ins v6.d[0], x22
|
||||
ins v7.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v1.d[1], x9
|
||||
ins v2.d[1], x11
|
||||
ins v3.d[1], x13
|
||||
ins v4.d[1], x15
|
||||
ins v5.d[1], x21
|
||||
ins v6.d[1], x23
|
||||
ins v7.d[1], x25
|
||||
|
||||
add x1, x1, #192
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
ld1 {{ v9.4s }}, [ x1 ], #16
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
|
||||
fmla v31.4s, v9.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x8, [x2], #8
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr d12, [x1], #8
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x12, [x1], #8
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr d14, [x1], #8
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x14, [x1], #8
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr d15, [x1], #8
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
ldr x15, [x1], #8
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.d[1], x8
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
ins v12.d[1], x12
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
ins v13.d[1], x13
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
ins v14.d[1], x14
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
ins v15.d[1], x15
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
ldr d14, [x1], #8
|
||||
|
||||
fmla v16.4s, v1.4s, v8.s[2]
|
||||
ldr x14, [x1], #8
|
||||
fmla v17.4s, v2.4s, v8.s[2]
|
||||
ldr d15, [x1], #8
|
||||
fmla v18.4s, v3.4s, v8.s[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v19.4s, v4.4s, v8.s[2]
|
||||
|
||||
ld1 {{ v0.4s }}, [x1], #16
|
||||
|
||||
fmla v20.4s, v5.4s, v8.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v8.s[2]
|
||||
ldr x10, [x1], #8
|
||||
|
||||
fmla v22.4s, v7.4s, v8.s[2]
|
||||
|
||||
fmla v23.4s, v9.4s, v8.s[2]
|
||||
ins v13.d[1], x13
|
||||
fmla v24.4s, v10.4s, v8.s[2]
|
||||
ins v14.d[1], x14
|
||||
fmla v25.4s, v11.4s, v8.s[2]
|
||||
ins v15.d[1], x15
|
||||
|
||||
fmla v26.4s, v12.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
fmla v27.4s, v13.4s, v8.s[2]
|
||||
ins v1.d[1], x10
|
||||
fmla v28.4s, v14.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
fmla v29.4s, v15.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
fmla v30.4s, v0.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
fmla v31.4s, v1.4s, v8.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.s[0], v8.s[3]
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
// load a: v9, v10, v11, v12, v13, v14, v15
|
||||
// load a: v0, v1, v2, v3, v4, v4, v6, v7
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v2.4s, v3.4s, v4.4s, v5.4s }}, [x1], #64
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v6.4s, v7.4s }}, [x1], #32
|
||||
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
fmla v16.4s, v1.4s, v8.s[0]
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
fmla v17.4s, v2.4s, v8.s[0]
|
||||
fmla v18.4s, v3.4s, v8.s[0]
|
||||
fmla v19.4s, v4.4s, v8.s[0]
|
||||
|
||||
fmla v20.4s, v5.4s, v8.s[0]
|
||||
fmla v21.4s, v6.4s, v8.s[0]
|
||||
fmla v22.4s, v7.4s, v8.s[0]
|
||||
fmla v23.4s, v9.4s, v8.s[0]
|
||||
|
||||
fmla v24.4s, v10.4s, v8.s[0]
|
||||
fmla v25.4s, v11.4s, v8.s[0]
|
||||
fmla v26.4s, v12.4s, v8.s[0]
|
||||
fmla v27.4s, v13.4s, v8.s[0]
|
||||
fmla v28.4s, v14.4s, v8.s[0]
|
||||
fmla v29.4s, v15.4s, v8.s[0]
|
||||
fmla v30.4s, v0.4s, v8.s[0]
|
||||
fmla v31.4s, v1.4s, v8.s[0]
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_64x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_64x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
sub x3, x3, #1
|
||||
|
||||
|
||||
ld1 { v8.s }[0], [ x2 ], #4
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop1/cortex_a53.tmpli" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop1/naive.tmpli" %}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
{% capture packed_packed_loop2 %}
|
||||
{% if core == "a53" %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% elsif core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop2/cortex_a55.tmpli" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop2/naive.tmpli" %}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
// last loop can't read beyond actual input as it's likely not packed and padded
|
||||
.packed_packed_loop_1_last:
|
||||
ld1 { v9.4s, v10.4s, v11.4s, v12.4s }, [x1], #64
|
||||
ld1 { v13.4s, v14.4s, v15.4s }, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ld1 { v0.4s }, [ x1 ]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_f32_scalars.tmpliq" from:16, to:31%}
|
||||
{% include "arm64simd_mmm_f32_per_rows.tmpliq" mr:64, from:16, to:31%}
|
||||
{% include "arm64simd_mmm_f32_per_cols.tmpliq" mr:64, from:16, to:31%}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #4
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.4s-v3.4s}, [x5], #64
|
||||
ld1 {v4.4s-v7.4s}, [x5], #64
|
||||
ld1 {v8.4s-v11.4s}, [x5], #64
|
||||
ld1 {v12.4s-v15.4s}, [x5], #64
|
||||
|
||||
{% for r in (0..15) %}
|
||||
fadd v{{r| plus: 16}}.4s, v{{r | plus: 16}}.4s, v{{r}}.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.s}[0], [ x3 ]
|
||||
|
||||
{% for r in (0..7) %}
|
||||
ldr q{{r}}, [x2], #16
|
||||
{% endfor %}
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr q0, [x2], #16
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr q1, [x2], #16
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr q2, [x2], #16
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr q3, [x2], #16
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr q4, [x2], #16
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr q5, [x2], #16
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr q6, [x2], #16
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr q7, [x2], #16
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc$
|
||||
|
||||
cmp x6, #4
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
st1 { v{{reg}}.4s }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
ld1 {{ v2.4s, v3.4s }}, [x1], #32
|
||||
ld1 {{ v6.4s, v7.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
and v0.16b, v2.16b, v2.16b
|
||||
and v1.16b, v3.16b, v3.16b
|
||||
and v4.16b, v6.16b, v6.16b
|
||||
and v5.16b, v7.16b, v7.16b
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v20.4s, v1.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr w22, [x2], #4
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr w8, [x1], #4
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr w23, [x2], #4
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
ldr w9, [x1], #4
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
ldr w24, [x2], #4
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
ldr w10, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
ldr w25, [x2], #4
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ldr w11, [x1], #4
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ldr w26, [x2], #4
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ldr w12, [x1], #4
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ldr w27, [x2], #4
|
||||
|
||||
ins v0.s[0], w5
|
||||
ins v4.s[0], w20
|
||||
ins v1.s[0], w9
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[2], w7
|
||||
ins v4.s[2], w22
|
||||
ins v1.s[2], w11
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[1], w6
|
||||
ins v4.s[1], w21
|
||||
ins v1.s[1], w10
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[3], w8
|
||||
ins v4.s[3], w23
|
||||
ins v1.s[3], w12
|
||||
ins v5.s[3], w27
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr w22, [x2], #4
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr w8, [x1], #4
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr w23, [x2], #4
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
ldr w9, [x1], #4
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
ldr w24, [x2], #4
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
ldr w10, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
ldr w25, [x2], #4
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ldr w11, [x1], #4
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ldr w26, [x2], #4
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ldr w12, [x1], #4
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ldr w27, [x2], #4
|
||||
|
||||
prfm pldl1keep, [x1, #256]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.s[0], w5
|
||||
ins v4.s[0], w20
|
||||
ins v1.s[0], w9
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[2], w7
|
||||
ins v4.s[2], w22
|
||||
ins v1.s[2], w11
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[1], w6
|
||||
ins v4.s[1], w21
|
||||
ins v1.s[1], w10
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[3], w8
|
||||
ins v4.s[3], w23
|
||||
ins v1.s[3], w12
|
||||
ins v5.s[3], w27
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr x11, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x8, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x12, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ins v2.d[0], x5
|
||||
ins v6.d[0], x9
|
||||
ins v3.d[0], x7
|
||||
ins v7.d[0], x11
|
||||
ins v2.d[1], x6
|
||||
ins v6.d[1], x10
|
||||
ins v3.d[1], x8
|
||||
ins v7.d[1], x12
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr x11, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x8, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x12, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #384]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #448]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #448]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v4.d[0], x9
|
||||
ins v1.d[0], x7
|
||||
ins v5.d[0], x11
|
||||
ins v0.d[1], x6
|
||||
ins v4.d[1], x10
|
||||
ins v1.d[1], x8
|
||||
ins v5.d[1], x12
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
ld1 {{ v2.4s, v3.4s }}, [x1], #32
|
||||
ld1 {{ v6.4s, v7.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v2.4s, v6.s[0]
|
||||
fmla v17.4s, v3.4s, v6.s[0]
|
||||
fmla v18.4s, v2.4s, v6.s[1]
|
||||
fmla v19.4s, v3.4s, v6.s[1]
|
||||
fmla v20.4s, v2.4s, v6.s[2]
|
||||
fmla v21.4s, v3.4s, v6.s[2]
|
||||
fmla v22.4s, v2.4s, v6.s[3]
|
||||
fmla v23.4s, v3.4s, v6.s[3]
|
||||
|
||||
fmla v24.4s, v2.4s, v7.s[0]
|
||||
fmla v25.4s, v3.4s, v7.s[0]
|
||||
fmla v26.4s, v2.4s, v7.s[1]
|
||||
fmla v27.4s, v3.4s, v7.s[1]
|
||||
fmla v28.4s, v2.4s, v7.s[2]
|
||||
fmla v29.4s, v3.4s, v7.s[2]
|
||||
fmla v30.4s, v2.4s, v7.s[3]
|
||||
fmla v31.4s, v3.4s, v7.s[3]
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d2, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d6, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr d7, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x8, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ins v2.d[1], x5
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ins v6.d[1], x7
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ins v3.d[1], x6
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ins v7.d[1], x8
|
||||
|
||||
fmla v16.4s, v2.4s, v6.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v3.4s, v6.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v2.4s, v6.s[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v6.s[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.4s, v2.4s, v6.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v3.4s, v6.s[2]
|
||||
ldr d5, [x2], #8
|
||||
fmla v22.4s, v2.4s, v6.s[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.4s, v3.4s, v6.s[3]
|
||||
ldr x8, [x2], #8
|
||||
|
||||
fmla v24.4s, v2.4s, v7.s[0]
|
||||
fmla v25.4s, v3.4s, v7.s[0]
|
||||
fmla v26.4s, v2.4s, v7.s[1]
|
||||
fmla v27.4s, v3.4s, v7.s[1]
|
||||
fmla v28.4s, v2.4s, v7.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v29.4s, v3.4s, v7.s[2]
|
||||
ins v4.d[1], x7
|
||||
fmla v30.4s, v2.4s, v7.s[3]
|
||||
ins v1.d[1], x6
|
||||
fmla v31.4s, v3.4s, v7.s[3]
|
||||
ins v5.d[1], x8
|
||||
|
||||
+181
@@ -0,0 +1,181 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_8x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_8x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed:
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
{% capture packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop1/ldr_x_preload.tmpli" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop1/naive.tmpli" %}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
{% capture packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop2/cortex_a55.tmpli" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endcapture %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_f32_scalars.tmpliq" from:16, to:31%}
|
||||
{% include "arm64simd_mmm_f32_per_rows.tmpliq" mr:8, from:16, to:31 %}
|
||||
{% include "arm64simd_mmm_f32_per_cols.tmpliq" mr:8, from:16, to:31 %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{col | times:2 | plus: reg}}.4s, v{{col | times:2 | plus: reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in (8..15) %}
|
||||
str q{{col | times:2 }}, [ x5 ]
|
||||
str q{{col | times:2 | plus: 1}}, [ x5, #16 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{col | times:2 | plus: reg}}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_min", op:"fmin", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_max", op:"fmax", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_mul", op:"fmul", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_add", op:"fadd", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub", op:"fsub", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_min", op:"fmin", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_max", op:"fmax", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_mul", op:"fmul", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_add", op:"fadd", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub", op:"fsub", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub_flipped", op:"fsub", mr:mr, from:from, to:to, flipped: true%}
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_min", op:"fmin", from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_max", op:"fmax", from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_mul", op:"fmul", from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_add", op:"fadd", from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_sub", op:"fsub", from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_sub_flipped", op:"fsub", from:from, to:to, flipped:true %}
|
||||
|
||||
.clear:
|
||||
{% for r in (from..to) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_i32_64x1_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_64x1_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
ld1 {v9.b}[0], [ x2 ], 1
|
||||
sshll v9.8h, v9.8b, 0
|
||||
|
||||
ld1 { v0.8b-v3.8b }, [ x1 ], #32
|
||||
ld1 { v4.8b-v7.8b }, [ x1 ], #32
|
||||
|
||||
{% for reg in (0..7) %}
|
||||
sshll v10.8h, v{{reg}}.8b, 0
|
||||
smlal v{{reg | times: 2 | plus: 16 }}.4s, v10.4h, v9.h[0]
|
||||
smlal2 v{{reg | times: 2 | plus: 17 }}.4s, v10.8h, v9.h[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
add v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v15.s }[0], [ x3 ]
|
||||
xtn v15.4h, v15.4s
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x2 ], #64
|
||||
|
||||
{% for reg in (0..7) %}
|
||||
xtn v{{reg}}.4h, v{{reg}}.4s
|
||||
smlal v{{reg|plus: 16}}.4s, v{{reg}}.4h, v15.h[0]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x2 ], #64
|
||||
|
||||
{% for reg in (0..7) %}
|
||||
xtn v{{reg}}.4h, v{{reg}}.4s
|
||||
smlal v{{reg|plus: 24}}.4s, v{{reg}}.4h, v15.h[0]
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_i32_scalars.tmpliq" from:16, to:31 %}
|
||||
{% include "arm64simd_mmm_i32_per_rows.tmpliq" mr:64, from:16, to:31 %}
|
||||
{% include "arm64simd_mmm_i32_per_cols.tmpliq" mr:64, from:16, to:31 %}
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.tmpliq" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{reg}}.b }[{{lane | times: 4}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for reg in (16..31) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_i32_8x8_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_8x8_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.tmpliq" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
ld1 { v0.8b }, [ x1 ], #8
|
||||
sshll v0.8h, v0.8b, 0
|
||||
ld1 { v4.8b }, [ x2 ], #8
|
||||
sshll v4.8h, v4.8b, 0
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal2 v17.4s, v0.8h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal2 v19.4s, v0.8h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal2 v21.4s, v0.8h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal2 v23.4s, v0.8h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v4.h[4]
|
||||
smlal2 v25.4s, v0.8h, v4.h[4]
|
||||
smlal v26.4s, v0.4h, v4.h[5]
|
||||
smlal2 v27.4s, v0.8h, v4.h[5]
|
||||
smlal v28.4s, v0.4h, v4.h[6]
|
||||
smlal2 v29.4s, v0.8h, v4.h[6]
|
||||
smlal v30.4s, v0.4h, v4.h[7]
|
||||
smlal2 v31.4s, v0.8h, v4.h[7]
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_i32_scalars.tmpliq" from:16, to:31%}
|
||||
{% include "arm64simd_mmm_i32_per_rows.tmpliq" mr:8, from:16, to:31%}
|
||||
{% include "arm64simd_mmm_i32_per_cols.tmpliq" mr:8, from:16, to:31%}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{col | times:2 | plus: reg}}.4s, v{{col | times:2 | plus: reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..3) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
add v{{col | times:2 | plus: reg}}.4s, v{{col | times:2 | plus: reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
xtn v0.4h, v0.4s
|
||||
xtn v1.4h, v1.4s
|
||||
xtn v4.4h, v4.4s
|
||||
xtn v5.4h, v5.4s
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal v17.4s, v1.4h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal v19.4s, v1.4h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal v21.4s, v1.4h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal v23.4s, v1.4h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v5.h[0]
|
||||
smlal v25.4s, v1.4h, v5.h[0]
|
||||
smlal v26.4s, v0.4h, v5.h[1]
|
||||
smlal v27.4s, v1.4h, v5.h[1]
|
||||
smlal v28.4s, v0.4h, v5.h[2]
|
||||
smlal v29.4s, v1.4h, v5.h[2]
|
||||
smlal v30.4s, v0.4h, v5.h[3]
|
||||
smlal v31.4s, v1.4h, v5.h[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.tmpliq" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{col | times:2 | plus: reg}}.b }[{{lane|times:4}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for col in (8..15) %}
|
||||
mov x4, x5
|
||||
{% for reg in (0..1) %}
|
||||
{% for lane in (0..3) %}
|
||||
st1 { v{{col | times:2 | plus: reg}}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_min", op:"smin", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_max", op:"smax", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_mul", op:"mul", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_add", op:"add", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub", op:"sub", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_col.tmpliq" label:"per_col_sub_flipped", op:"sub", mr:mr, from:from, to:to, flipped: true %}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_min", op:"smin", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_max", op:"smax", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_mul", op:"mul", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_add", op:"add", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub", op:"sub", mr:mr, from:from, to:to %}
|
||||
{% include "arm64simd_mmm_4s_per_row.tmpliq" label:"per_row_sub_flipped", op:"sub", mr:mr, from:from, to:to, flipped: true %}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_min", op:"smin", from:from, to:to%}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_max", op:"smax", from:from, to:to%}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_mul", op:"mul", from:from, to:to%}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_add", op:"add", from:from, to:to%}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_sub", op:"sub", from:from, to:to%}
|
||||
{% include "arm64simd_mmm_4s_scalar.tmpliq" label:"scalar_sub_flipped", op:"sub", from:from, to:to, flipped:true%}
|
||||
|
||||
.clear:
|
||||
{% for r in (from..to) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
|
||||
// vim: ft=arm
|
||||
|
||||
.q_scale:
|
||||
ldp x5, x6, [x0, #8] // x5: shift, x6: policy
|
||||
add x2, x0, #24
|
||||
ld1r { v2.4s }, [x2] // v2.4s <- multiplier
|
||||
|
||||
mov w3, #1
|
||||
ins v4.d[0], x3
|
||||
dup v4.2d, v4.d[0] // v4.2d <- 1
|
||||
|
||||
add x5, x5, #32 // add 32 to shift
|
||||
neg x5, x5 // broadcast shift
|
||||
ins v1.d[0], x5
|
||||
dup v1.2d, v1.d[0] // v1.2s <- -(shift + 32)
|
||||
|
||||
cmp x6, 1
|
||||
beq .q_scale_rounding_zero
|
||||
cmp x6, 2
|
||||
beq .q_scale_rounding_away
|
||||
cmp x6, 3
|
||||
beq .q_scale_rounding_minus_inf
|
||||
cmp x6, 4
|
||||
beq .q_scale_rounding_plus_inf
|
||||
cmp x6, 5
|
||||
beq .q_scale_rounding_even
|
||||
cmp x6, 6
|
||||
beq .q_scale_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_scale_rounding_zero:
|
||||
// rust: signum * ((abs + nudge2) >> shift
|
||||
// asm: signum * (2*abs - 1) >>r (shift + 1)
|
||||
|
||||
{% for q in (16..31) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sub v8.2d, v8.2d, v4.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sub v9.2d, v9.2d, v4.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_away: // signum * (abs >> (shift-1) + 1 >> 1)
|
||||
|
||||
{% for q in (16..31) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_minus_inf: // val >> shift
|
||||
|
||||
{% for q in (16..31) %}
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sub v8.2d, v8.2d, v4.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sub v9.2d, v9.2d, v4.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_plus_inf: // (val >> shift-1)+1 >>1
|
||||
|
||||
{% for q in (16..31) %}
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_even: // signum * ((abs >> shift-1) + (abs & 0x1) - 1 >> 1)
|
||||
|
||||
{% for q in (16..31) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqshl v3.2d, v8.2d, v1.2d // abs >> shift - 1
|
||||
and v3.16b, v3.16b, v4.16b // abs & 0x1
|
||||
sub v3.2d, v3.2d, v4.2d //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
add v8.2d, v8.2d, v3.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sqshl v3.2d, v9.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
sub v3.2d, v3.2d, v4.2d //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
add v9.2d, v9.2d, v3.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_odd: // signum * ((abs >> shift-1) - (abs & 0x1) >> 1)
|
||||
|
||||
{% for q in (16..31) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqshl v3.2d, v8.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
sub v8.2d, v8.2d, v3.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sqshl v3.2d, v9.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
sub v9.2d, v9.2d, v3.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
ldr x5, [x0, #8] // x5: shift
|
||||
ins v1.s[0], w5
|
||||
dup v1.4s, v1.s[0] // v1.4s <- shift
|
||||
|
||||
{% for q in (16..31) %}
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr:
|
||||
ldp x5, x6, [x0, #8] // x5: shift, x6: policy
|
||||
|
||||
mov w3, #1
|
||||
ins v4.s[0], w3
|
||||
dup v4.4s, v4.s[0] // v4.4d <- 1
|
||||
|
||||
neg w5, w5 // broadcast shift
|
||||
ins v1.s[0], w5
|
||||
dup v1.4s, v1.s[0] // v1.4s <- -shift
|
||||
|
||||
cmp x6, 1
|
||||
beq .q_shr_rounding_zero
|
||||
cmp x6, 2
|
||||
beq .q_shr_rounding_away
|
||||
cmp x6, 3
|
||||
beq .q_shr_rounding_minus_inf
|
||||
cmp x6, 4
|
||||
beq .q_shr_rounding_plus_inf
|
||||
cmp x6, 5
|
||||
beq .q_shr_rounding_even
|
||||
cmp x6, 6
|
||||
beq .q_shr_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_shr_rounding_zero:
|
||||
// asm: signum * (abs >>r shift)
|
||||
{% for q in (16..31) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
|
||||
sub v{{q}}.4s, v{{q}}.4s, v4.4s
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_away:
|
||||
{% for q in (16..31) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_minus_inf:
|
||||
{% for q in (16..31) %}
|
||||
sqneg v{{q}}.4s, v{{q}}.4s
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
sqneg v{{q}}.4s, v{{q}}.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_plus_inf:
|
||||
{% for q in (16..31) %}
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_even:
|
||||
// sqrshl is round(+inf), sqshl trauncates
|
||||
// we look at parity of result by truncation: if it's odd, we have nothing more to do, we go towards +inf
|
||||
// if it's even, we need to nudge towards 0 by adding -1
|
||||
// => nudge = (x >>l shift) & 0x1 - 1 (>>l is sqshl)
|
||||
// => result is (x + nudge) >>r shift (with sqrshl)
|
||||
{% for q in (16..31) %}
|
||||
sqshl v3.4s, v{{q}}.4s, v1.4s // trunc
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
sub v3.4s, v3.4s, v4.4s
|
||||
add v{{q}}.4s, v{{q}}.4s, v3.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_odd:
|
||||
// here: nudge is -((x >>l shift) & 0x1)
|
||||
{% for q in (16..31) %}
|
||||
sqshl v3.4s, v{{q}}.4s, v1.4s // trunc
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
neg v3.4s, v3.4s
|
||||
add v{{q}}.4s, v{{q}}.4s, v3.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_sigmoid_f32_4n_{{suffix}}
|
||||
{{G}}arm64simd_sigmoid_f32_4n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
dup v5.4s, v0.s[0] // v5 <- low, broadcasted
|
||||
dup v6.4s, v0.s[1] // v6 <- high, broadcasted
|
||||
dup v7.4s, v3.s[1] // v7 <- 0.5, broadcasted
|
||||
|
||||
cmp x1, #16
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmax v17.4s, v17.4s, v5.4s
|
||||
fmax v18.4s, v18.4s, v5.4s
|
||||
fmax v19.4s, v19.4s, v5.4s
|
||||
|
||||
fmin v16.4s, v16.4s, v6.4s
|
||||
fmin v17.4s, v17.4s, v6.4s
|
||||
fmin v18.4s, v18.4s, v6.4s
|
||||
fmin v19.4s, v19.4s, v6.4s // v16 <- x
|
||||
|
||||
fmul v20.4s, v16.4s, v16.4s
|
||||
fmul v21.4s, v17.4s, v17.4s
|
||||
fmul v22.4s, v18.4s, v18.4s
|
||||
fmul v23.4s, v19.4s, v19.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v25.4s, v0.s[3]
|
||||
fmla v25.4s, v21.4s, v0.s[2]
|
||||
dup v26.4s, v0.s[3]
|
||||
fmla v26.4s, v22.4s, v0.s[2]
|
||||
dup v27.4s, v0.s[3]
|
||||
fmla v27.4s, v23.4s, v0.s[2]
|
||||
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[1]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[1]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[1]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[2]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[2]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[2]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[3]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[3]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[3]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
fmul v16.4s, v16.4s, v28.4s
|
||||
fmul v17.4s, v17.4s, v29.4s
|
||||
fmul v18.4s, v18.4s, v30.4s
|
||||
fmul v19.4s, v19.4s, v31.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v25.4s, v2.s[2]
|
||||
fmla v25.4s, v21.4s, v2.s[1]
|
||||
dup v26.4s, v2.s[2]
|
||||
fmla v26.4s, v22.4s, v2.s[1]
|
||||
dup v27.4s, v2.s[2]
|
||||
fmla v27.4s, v23.4s, v2.s[1]
|
||||
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[3]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[3]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[3]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v3.s[0]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v3.s[0]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v3.s[0]
|
||||
fmla v27.4s, v23.4s, v31.4s // v24 denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
fdiv v17.4s, v17.4s, v25.4s
|
||||
fdiv v18.4s, v18.4s, v26.4s
|
||||
fdiv v19.4s, v19.4s, v27.4s
|
||||
|
||||
fadd v16.4s, v16.4s, v7.4s
|
||||
fadd v17.4s, v17.4s, v7.4s
|
||||
fadd v18.4s, v18.4s, v7.4s
|
||||
fadd v19.4s, v19.4s, v7.4s
|
||||
|
||||
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
|
||||
|
||||
subs x1, x1, #16
|
||||
cmp x1, #16
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmin v16.4s, v16.4s, v6.4s // v16 <- x
|
||||
fmul v20.4s, v16.4s, v16.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
fmul v16.4s, v16.4s, v28.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s // v24 <- denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
fadd v16.4s, v16.4s, v7.4s
|
||||
|
||||
st1 { v16.4s }, [x0], #16
|
||||
|
||||
subs x1, x1, #4
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
.float -18.6 // low
|
||||
.float 18.6 // high
|
||||
.float -4.433153405e-18 // alpha_13
|
||||
.float 1.169974371e-14
|
||||
|
||||
.float -1.875289645e-11
|
||||
.float 4.257889523e-8
|
||||
.float 0.00004811817576
|
||||
.float 0.008163842030
|
||||
|
||||
.float 0.2499999971
|
||||
.float 3.922935744e-6 // beta_6
|
||||
.float 0.001524872358
|
||||
.float 0.1159886749
|
||||
|
||||
.float 1.0
|
||||
.float 0.5 //
|
||||
.float 0.0 // padding
|
||||
.float 0.0
|
||||
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_tanh_f32_4n_{{suffix}}
|
||||
{{G}}arm64simd_tanh_f32_4n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
dup v5.4s, v0.s[0] // v5 <- low, broadcasted
|
||||
dup v6.4s, v0.s[1] // v6 <- high, broadcasted
|
||||
|
||||
cmp x1, #16
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmax v17.4s, v17.4s, v5.4s
|
||||
fmax v18.4s, v18.4s, v5.4s
|
||||
fmax v19.4s, v19.4s, v5.4s
|
||||
|
||||
fmin v16.4s, v16.4s, v6.4s
|
||||
fmin v17.4s, v17.4s, v6.4s
|
||||
fmin v18.4s, v18.4s, v6.4s
|
||||
fmin v19.4s, v19.4s, v6.4s // v16 <- x
|
||||
|
||||
fmul v20.4s, v16.4s, v16.4s
|
||||
fmul v21.4s, v17.4s, v17.4s
|
||||
fmul v22.4s, v18.4s, v18.4s
|
||||
fmul v23.4s, v19.4s, v19.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v25.4s, v0.s[3]
|
||||
fmla v25.4s, v21.4s, v0.s[2]
|
||||
dup v26.4s, v0.s[3]
|
||||
fmla v26.4s, v22.4s, v0.s[2]
|
||||
dup v27.4s, v0.s[3]
|
||||
fmla v27.4s, v23.4s, v0.s[2]
|
||||
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[1]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[1]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[1]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v1.s[2]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v1.s[2]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v1.s[2]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v1.s[3]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v1.s[3]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v1.s[3]
|
||||
fmla v27.4s, v23.4s, v31.4s
|
||||
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[0]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[0]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[0]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
fmul v16.4s, v16.4s, v28.4s
|
||||
fmul v17.4s, v17.4s, v29.4s
|
||||
fmul v18.4s, v18.4s, v30.4s
|
||||
fmul v19.4s, v19.4s, v31.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v25.4s, v2.s[2]
|
||||
fmla v25.4s, v21.4s, v2.s[1]
|
||||
dup v26.4s, v2.s[2]
|
||||
fmla v26.4s, v22.4s, v2.s[1]
|
||||
dup v27.4s, v2.s[2]
|
||||
fmla v27.4s, v23.4s, v2.s[1]
|
||||
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v29.4s, v2.s[3]
|
||||
fmla v29.4s, v21.4s, v25.4s
|
||||
dup v30.4s, v2.s[3]
|
||||
fmla v30.4s, v22.4s, v26.4s
|
||||
dup v31.4s, v2.s[3]
|
||||
fmla v31.4s, v23.4s, v27.4s
|
||||
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v25.4s, v3.s[0]
|
||||
fmla v25.4s, v21.4s, v29.4s
|
||||
dup v26.4s, v3.s[0]
|
||||
fmla v26.4s, v22.4s, v30.4s
|
||||
dup v27.4s, v3.s[0]
|
||||
fmla v27.4s, v23.4s, v31.4s // v24 denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
fdiv v17.4s, v17.4s, v25.4s
|
||||
fdiv v18.4s, v18.4s, v26.4s
|
||||
fdiv v19.4s, v19.4s, v27.4s
|
||||
|
||||
st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
|
||||
|
||||
subs x1, x1, #16
|
||||
cmp x1, #16
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.4s }, [x0]
|
||||
|
||||
fmax v16.4s, v16.4s, v5.4s
|
||||
fmin v16.4s, v16.4s, v6.4s // v16 <- x
|
||||
fmul v20.4s, v16.4s, v16.4s // v20 <- x2
|
||||
|
||||
dup v24.4s, v0.s[3]
|
||||
fmla v24.4s, v20.4s, v0.s[2]
|
||||
dup v28.4s, v1.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[1]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v1.s[2]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v1.s[3]
|
||||
fmla v24.4s, v20.4s, v28.4s
|
||||
dup v28.4s, v2.s[0]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
fmul v16.4s, v16.4s, v28.4s // v16 <- numerator
|
||||
|
||||
dup v24.4s, v2.s[2]
|
||||
fmla v24.4s, v20.4s, v2.s[1]
|
||||
dup v28.4s, v2.s[3]
|
||||
fmla v28.4s, v20.4s, v24.4s
|
||||
dup v24.4s, v3.s[0]
|
||||
fmla v24.4s, v20.4s, v28.4s // v24 <- denum
|
||||
|
||||
fdiv v16.4s, v16.4s, v24.4s
|
||||
|
||||
st1 { v16.4s }, [x0], #16
|
||||
|
||||
subs x1, x1, #4
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
.float -8.9 // low
|
||||
.float 8.9 // high
|
||||
.float -8.488492677e-14 // alpha_13
|
||||
.float 5.277853000e-11
|
||||
|
||||
.float -2.022500419e-8
|
||||
.float 0.00001115424833
|
||||
.float 0.003103950131
|
||||
.float 0.1308400453
|
||||
|
||||
.float 0.9999999934
|
||||
.float 0.0002546136580 // beta_6
|
||||
.float 0.02449515379
|
||||
.float 0.4641733162
|
||||
|
||||
.float 1.0
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
sub x0, x0, 40
|
||||
|
||||
.non_linear_loop:
|
||||
add x0, x0, 40
|
||||
ldr x2, [x0]
|
||||
|
||||
mov x4, #{{ jump_table | size }}
|
||||
|
||||
cmp x2, #{{ jump_table | size }}
|
||||
csel x2, x2, x4, lt
|
||||
cmp x2, #0
|
||||
csel x2, x4, x2, lt
|
||||
|
||||
adr x3, .jmp_table
|
||||
add x3, x3, x2, LSL#2
|
||||
br x3
|
||||
|
||||
.jmp_table:
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
add x0, x2, #4000
|
||||
b .return
|
||||
|
||||
.unsupported:
|
||||
mov x0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov x0, 0
|
||||
b .return
|
||||
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
#![feature(asm)]
|
||||
#![allow(
|
||||
dead_code,
|
||||
non_upper_case_globals,
|
||||
unused_macros,
|
||||
non_snake_case,
|
||||
unused_assignments
|
||||
)]
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
macro_rules! r2 { ($($stat:stmt)*) => { $( $stat )* $( $stat )* } }
|
||||
macro_rules! r4 { ($($stat:stmt)*) => { r2!(r2!($($stat)*)) }}
|
||||
macro_rules! r8 { ($($stat:stmt)*) => { r4!(r2!($($stat)*)) }}
|
||||
macro_rules! r16 { ($($stat:stmt)*) => { r4!(r4!($($stat)*)) }}
|
||||
macro_rules! r32 { ($($stat:stmt)*) => { r8!(r4!($($stat)*)) }}
|
||||
macro_rules! r64 { ($($stat:stmt)*) => { r8!(r8!($($stat)*)) }}
|
||||
macro_rules! r128 { ($($stat:stmt)*) => { r8!(r16!($($stat)*)) }}
|
||||
macro_rules! r1024 { ($($stat:stmt)*) => { r8!(r128!($($stat)*)) }}
|
||||
macro_rules! r4096 { ($($stat:stmt)*) => { r4!(r1024!($($stat)*)) }}
|
||||
|
||||
const _F32: [f32; 1024] = [12.; 1024];
|
||||
const F32: *const f32 = _F32.as_ptr();
|
||||
|
||||
/*
|
||||
fn ruin_cache() {
|
||||
let _a = (0..1000000).collect::<Vec<i32>>();
|
||||
}
|
||||
*/
|
||||
|
||||
macro_rules! b {
|
||||
($f: block, $inner_loop: expr, $measures: expr) => {{
|
||||
let mut values = Vec::with_capacity($measures);
|
||||
for _ in 0..$measures {
|
||||
// ruin_cache();
|
||||
let start = Instant::now();
|
||||
for _ in 0..$inner_loop {
|
||||
unsafe { $f };
|
||||
}
|
||||
values.push(start.elapsed());
|
||||
}
|
||||
values.sort();
|
||||
values[$measures / 2].as_nanos() as f64 / 1e9 / $inner_loop as f64
|
||||
}};
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let cycle = b!(
|
||||
{
|
||||
r1024!(asm!("orr r0, r0, r0", out("r0") _));
|
||||
},
|
||||
1000,
|
||||
1000
|
||||
) / 1024.;
|
||||
let indep_fmla = b!(
|
||||
{
|
||||
r8!(asm!("
|
||||
vmla.f32 q0, q0, q0
|
||||
vmla.f32 q1, q1, q1
|
||||
vmla.f32 q2, q2, q2
|
||||
vmla.f32 q3, q3, q3
|
||||
vmla.f32 q4, q4, q4
|
||||
vmla.f32 q5, q5, q5
|
||||
vmla.f32 q6, q6, q6
|
||||
vmla.f32 q7, q7, q7
|
||||
", out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _));
|
||||
},
|
||||
1000,
|
||||
1000
|
||||
) / 64.;
|
||||
eprintln!("rcp tp: indep fmla: {}", indep_fmla / cycle);
|
||||
let dep_accu_fmla = b!(
|
||||
{
|
||||
r16!(asm!("
|
||||
vmla.f32 q15, q0, q0
|
||||
vmla.f32 q15, q1, q1
|
||||
vmla.f32 q15, q2, q2
|
||||
vmla.f32 q15, q3, q3
|
||||
vmla.f32 q15, q4, q4
|
||||
vmla.f32 q15, q5, q5
|
||||
vmla.f32 q15, q6, q6
|
||||
vmla.f32 q15, q7, q7
|
||||
vmla.f32 q15, q8, q8
|
||||
vmla.f32 q15, q9, q9
|
||||
vmla.f32 q15, q10, q10
|
||||
vmla.f32 q15, q11, q11
|
||||
vmla.f32 q15, q12, q12
|
||||
vmla.f32 q15, q13, q13
|
||||
vmla.f32 q15, q14, q14
|
||||
", out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
|
||||
out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
|
||||
},
|
||||
1000,
|
||||
1000
|
||||
) / 16.
|
||||
/ 15.;
|
||||
eprintln!("rcp tp: accu-dep fmla: {}", dep_accu_fmla / cycle);
|
||||
let load_s_using_vld1_64 = b!(
|
||||
{
|
||||
let mut p = F32;
|
||||
r16!(asm!("
|
||||
vld1.64 {{d0-d3}}, [{0}]!
|
||||
vld1.64 {{d4-d7}}, [{0}]!
|
||||
vld1.64 {{d8-d11}}, [{0}]!
|
||||
vld1.64 {{d12-d15}}, [{0}]!
|
||||
vld1.64 {{d16-d19}}, [{0}]!
|
||||
vld1.64 {{d20-d23}}, [{0}]!
|
||||
vld1.64 {{d24-d27}}, [{0}]!
|
||||
vld1.64 {{d28-d31}}, [{0}]!
|
||||
",
|
||||
inout(reg) p,
|
||||
out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
|
||||
out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
|
||||
},
|
||||
1000,
|
||||
1000
|
||||
) / 16.
|
||||
/ 64.; // each line load 8 s
|
||||
eprintln!(
|
||||
"rcp tp: load s using vld1_64 ia {}",
|
||||
load_s_using_vld1_64 / cycle
|
||||
);
|
||||
let load_s_using_vldm_q = b!(
|
||||
{
|
||||
let mut p = F32;
|
||||
r16!(asm!("
|
||||
vldm {0}!, {{q0-q3}}
|
||||
vldm {0}!, {{q4-q7}}
|
||||
vldm {0}!, {{q8-q11}}
|
||||
vldm {0}!, {{q12-q15}}
|
||||
",
|
||||
inout(reg) p,
|
||||
out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
|
||||
out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
|
||||
},
|
||||
1000,
|
||||
1000
|
||||
) / 16.
|
||||
/ 64.;
|
||||
eprintln!(
|
||||
"rcp tp: load s using vldmia q: {}",
|
||||
load_s_using_vldm_q / cycle
|
||||
);
|
||||
let load = b!(
|
||||
{
|
||||
let mut p = F32;
|
||||
r16!(asm!("
|
||||
vldr.64 d0, [{0}]
|
||||
vldr.64 d1, [{0}, #8]
|
||||
vldr.64 d2, [{0}, #16]
|
||||
vldr.64 d3, [{0}, #24]
|
||||
vldr.64 d4, [{0}, #32]
|
||||
vldr.64 d5, [{0}, #40]
|
||||
vldr.64 d6, [{0}, #48]
|
||||
vldr.64 d7, [{0}, #56]
|
||||
vldr.64 d8, [{0}, #64]
|
||||
vldr.64 d9, [{0}, #72]
|
||||
vldr.64 d10, [{0}, #80]
|
||||
vldr.64 d11, [{0}, #88]
|
||||
vldr.64 d12, [{0}, #96]
|
||||
vldr.64 d13, [{0}, #104]
|
||||
vldr.64 d14, [{0}, #112]
|
||||
vldr.64 d15, [{0}, #120]
|
||||
vldr.64 d16, [{0}, #128]
|
||||
vldr.64 d17, [{0}, #136]
|
||||
vldr.64 d18, [{0}, #144]
|
||||
vldr.64 d19, [{0}, #152]
|
||||
vldr.64 d20, [{0}, #160]
|
||||
vldr.64 d21, [{0}, #168]
|
||||
vldr.64 d22, [{0}, #176]
|
||||
vldr.64 d23, [{0}, #184]
|
||||
vldr.64 d24, [{0}, #192]
|
||||
vldr.64 d25, [{0}, #200]
|
||||
vldr.64 d26, [{0}, #208]
|
||||
vldr.64 d27, [{0}, #216]
|
||||
vldr.64 d28, [{0}, #224]
|
||||
vldr.64 d29, [{0}, #232]
|
||||
vldr.64 d30, [{0}, #240]
|
||||
vldr.64 d31, [{0}, #248]
|
||||
add {0}, #256
|
||||
",
|
||||
inout(reg) p,
|
||||
out("q0") _, out("q1") _, out("q2") _, out("q3") _, out("q4") _, out("q5") _, out("q6") _, out("q7") _,
|
||||
out("q8") _, out("q9") _, out("q10") _, out("q11") _, out("q12") _, out("q13") _, out("q14") _, out("q15") _));
|
||||
},
|
||||
1000,
|
||||
1000
|
||||
) / 16.
|
||||
/ 64.;
|
||||
eprintln!("rcp tp: load s using vldr d + imm: {}", load / cycle);
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use tract_data::prelude::*;
|
||||
use tract_linalg::frame::mmm::LinearSpec;
|
||||
use tract_linalg::frame::mmm::MatMatMulKer;
|
||||
use tract_linalg::frame::mmm::MatMatMulKerSpec;
|
||||
use tract_linalg::mmm::{InputStoreKer, OutputStoreKer};
|
||||
|
||||
fn ruin_cache() {
|
||||
let _a = (0..1000000).collect::<Vec<i32>>();
|
||||
}
|
||||
|
||||
fn bench_to_nanos<T: Datum + Copy + num_traits::Zero, K: MatMatMulKer<T>>(
|
||||
k: usize,
|
||||
loops: usize,
|
||||
) -> f64 {
|
||||
let item_size = T::datum_type().size_of();
|
||||
let a = Tensor::zero_aligned::<T>(
|
||||
&[(k + K::end_padding_packed_a()) * K::mr()],
|
||||
K::alignment_bytes_packed_a(),
|
||||
)
|
||||
.unwrap();
|
||||
let b = Tensor::zero_aligned::<T>(
|
||||
&[(k + K::end_padding_packed_b()) * K::nr()],
|
||||
K::alignment_bytes_packed_b(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut c = Tensor::zero::<T>(&[K::mr() * K::nr()]).unwrap();
|
||||
let ref a = InputStoreKer::Packed {
|
||||
ptr: unsafe { a.as_ptr_unchecked::<u8>() as _ },
|
||||
};
|
||||
let ref b = InputStoreKer::Packed {
|
||||
ptr: unsafe { b.as_ptr_unchecked::<u8>() as _ },
|
||||
};
|
||||
let ref c = OutputStoreKer {
|
||||
ptr: unsafe { c.as_ptr_mut_unchecked::<u8>() as _ },
|
||||
item_size,
|
||||
col_byte_stride: (item_size * K::mr()) as isize,
|
||||
row_byte_stride: item_size as isize,
|
||||
};
|
||||
let ref linear = LinearSpec::Mul { k };
|
||||
let op = MatMatMulKerSpec {
|
||||
a,
|
||||
b,
|
||||
c,
|
||||
linear,
|
||||
non_linear: std::ptr::null(),
|
||||
};
|
||||
let mut values = Vec::with_capacity(loops);
|
||||
for _ in 0..loops {
|
||||
ruin_cache();
|
||||
let start = Instant::now();
|
||||
K::kernel(&op);
|
||||
values.push(start.elapsed());
|
||||
}
|
||||
values.sort();
|
||||
values[loops / 2].as_nanos() as f64
|
||||
}
|
||||
|
||||
fn model<T: Datum + Copy + num_traits::Zero, K: MatMatMulKer<T>>() -> (f64, f64) {
|
||||
let x = 1000;
|
||||
let zp = bench_to_nanos::<T, K>(0, 10000);
|
||||
let y = bench_to_nanos::<T, K>(x, 1000);
|
||||
let slope = (y - zp) / x as f64;
|
||||
(slope, zp)
|
||||
}
|
||||
|
||||
fn as_match_line<T: Datum + Copy + num_traits::Zero, K: MatMatMulKer<T>>() {
|
||||
let coeffs = model::<T, K>();
|
||||
println!(
|
||||
"({:?}, {}, {}) => {} * k + {},",
|
||||
K::name(),
|
||||
K::mr(),
|
||||
K::nr(),
|
||||
(coeffs.0 * 1000.).round(),
|
||||
(coeffs.1 * 1000.).round()
|
||||
);
|
||||
}
|
||||
|
||||
fn main() {
|
||||
use tract_linalg::arm64::*;
|
||||
as_match_line::<f32, MatMatMulF32x16x4>();
|
||||
as_match_line::<f32, MatMatMulF32x12x8>();
|
||||
as_match_line::<f32, MatMatMulF32x8x8>();
|
||||
as_match_line::<f32, MatMatMulF32x16x4A53>();
|
||||
as_match_line::<f32, MatMatMulF32x12x8A53>();
|
||||
as_match_line::<f32, MatMatMulF32x8x8A53>();
|
||||
}
|
||||
+1095
File diff suppressed because it is too large
Load Diff
Vendored
+52
@@ -0,0 +1,52 @@
|
||||
#[macro_use]
|
||||
extern crate criterion;
|
||||
extern crate tract_data;
|
||||
extern crate tract_linalg;
|
||||
use criterion::Criterion;
|
||||
use tract_linalg::frame::mmm::FusedSpec;
|
||||
|
||||
use tract_data::internal::*;
|
||||
use DatumType::F32;
|
||||
|
||||
fn conv(c: &mut Criterion, dilation: usize, pulse: usize, ci: usize, co: usize) {
|
||||
c.bench_function(
|
||||
&format!("conv_d{dilation}p{pulse}ci{ci}co{co}"),
|
||||
move |be| unsafe {
|
||||
let t = pulse + 2 * dilation;
|
||||
let k = ci * 3;
|
||||
let mm = tract_linalg::ops()
|
||||
.mmm(F32, F32, F32, Some(co), Some(ci * 3), Some(pulse))
|
||||
.unwrap();
|
||||
mm.c_from_data_and_strides(F32.size_of(), co, t, t as _, 1);
|
||||
let a = Tensor::zero_aligned::<f32>(&[mm.a_pack().len(k, co)], mm.a_pack().alignment())
|
||||
.unwrap();
|
||||
let input = Tensor::zero::<f32>(&[ci, t]).unwrap();
|
||||
let mut output = Tensor::zero::<f32>(&[co, t]).unwrap();
|
||||
be.iter(move || {
|
||||
mm.run(
|
||||
co,
|
||||
t,
|
||||
&[
|
||||
FusedSpec::AddMatMul {
|
||||
a: mm.a_packed(F32.size_of(), k).wrap(&a.view()),
|
||||
b: mm.b_packed(F32.size_of(), k).wrap(&input.view()).unwrap(),
|
||||
k,
|
||||
},
|
||||
FusedSpec::Store(mm.c_view(0, 1).wrap(&output.view_mut())),
|
||||
],
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn convs(c: &mut Criterion) {
|
||||
conv(c, 1, 8, 16, 64);
|
||||
conv(c, 2, 8, 16, 64);
|
||||
conv(c, 4, 8, 16, 64);
|
||||
conv(c, 8, 8, 16, 64);
|
||||
}
|
||||
|
||||
criterion_group!(benches, convs);
|
||||
criterion_main!(benches);
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
use criterion::*;
|
||||
use tract_data::internal::*;
|
||||
use tract_linalg::frame::mmm::FusedSpec;
|
||||
|
||||
use DatumType::F32;
|
||||
|
||||
fn mat_vec_mul(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("mat_vec_mul");
|
||||
unsafe {
|
||||
{
|
||||
let (m, k) = &(768usize, 256usize);
|
||||
group.throughput(Throughput::Elements((m * k) as u64));
|
||||
group.bench_with_input(
|
||||
BenchmarkId::from_parameter(format!("{m}x{k}")),
|
||||
&(m, k),
|
||||
|be, (&m, &k)| {
|
||||
let mm = tract_linalg::ops()
|
||||
.mmm(F32, F32, F32, Some(m), Some(k), Some(1))
|
||||
.unwrap();
|
||||
let pa = Tensor::uninitialized_aligned::<f32>(
|
||||
&[mm.a_pack().len(k, m)],
|
||||
mm.a_pack().alignment(),
|
||||
)
|
||||
.unwrap();
|
||||
let b = tensor1(&vec![0.0; k]);
|
||||
let mut c = Tensor::zero::<f32>(&[m]).unwrap();
|
||||
be.iter(move || {
|
||||
mm.run(
|
||||
m,
|
||||
1,
|
||||
&[
|
||||
FusedSpec::AddMatMul {
|
||||
a: mm.a_packed(F32.size_of(), k).wrap(&pa.view()),
|
||||
b: mm
|
||||
.b_packed(b.datum_type().size_of(), k)
|
||||
.wrap(&b.view())
|
||||
.unwrap(),
|
||||
k,
|
||||
},
|
||||
FusedSpec::Store(mm.c_view(0, 0).wrap(&c.view_mut())),
|
||||
],
|
||||
)
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, mat_vec_mul);
|
||||
criterion_main!(benches);
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
use criterion::*;
|
||||
|
||||
mod utils;
|
||||
use utils::*;
|
||||
|
||||
fn all(c: &mut Criterion) {
|
||||
// packed_packed: co, ci, n
|
||||
// direct_conv(c, "asr_2M", 24, 5, 40, 200, 1); // lda
|
||||
packed_packed(c, "asr_2M", 256, 200, 24); // tdnn1
|
||||
// direct_conv(c, "asr_2M", 24, 3, 256, 256, 1); // tdnn2
|
||||
// direct_conv(c, "asr_2M", 24, 3, 256, 256, 3); // tdnn3
|
||||
packed_packed(c, "asr_2M", 256, 256, 8); // fastlstm1 and 2 (input) x 8 (4 prod x 2 layers)
|
||||
packed_packed(c, "asr_2M", 256, 128, 1); // fastlstm1 and 2 (hidden) x 64 (4 prod x 2 layers x 8 loops)
|
||||
packed_packed(c, "asr_2M", 256, 256, 1); // fastlstm1 and 2 (rp) x 16 (2 layers x 8 loops)
|
||||
// direct_conv(c, "asr_2M", 8, 3, 256, 256, 1); // tdnn4, tdd5 (x2)
|
||||
packed_packed(c, "asr_2M", 1690, 256, 8); // output
|
||||
|
||||
// 8M
|
||||
packed_packed(c, "asr_8M", 512, 200, 24); // tdnn1
|
||||
packed_packed(c, "asr_8M", 512, 512, 24); // tdnn2
|
||||
packed_packed(c, "asr_8M", 512, 256, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
|
||||
packed_vec(c, "asr_8M", 512, 256, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
|
||||
|
||||
// pseudo 15M
|
||||
packed_packed(c, "asr_pseudo15M", 768, 200, 24); // tdnn1
|
||||
packed_packed(c, "asr_pseudo15M", 768, 2304, 24); // tdnn2
|
||||
packed_packed(c, "asr_pseudo15M", 768, 2304, 8); // tdnn3,4,5
|
||||
packed_packed(c, "asr_pseudo15M", 768, 768, 8); // fastlstm1 and 2 (four parts, rec mat*mat)
|
||||
packed_packed(c, "asr_pseudo15M", 768, 384, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
|
||||
packed_vec(c, "asr_pseudo15M", 768, 384, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
|
||||
|
||||
// 15M
|
||||
packed_vec(c, "asr_15M", 768, 256, 1); // fastlstm1 and 2 (four parts, rec mat*vec)
|
||||
}
|
||||
|
||||
criterion_group!(benches, all);
|
||||
criterion_main!(benches);
|
||||
Vendored
+47
@@ -0,0 +1,47 @@
|
||||
extern crate criterion;
|
||||
use criterion::*;
|
||||
use tract_data::internal::*;
|
||||
use tract_linalg::frame::mmm::FusedSpec;
|
||||
|
||||
use DatumType::F32;
|
||||
|
||||
fn mat_mul_smmm(be: &mut criterion::Bencher, &(m, k, n): &(usize, usize, usize)) {
|
||||
unsafe {
|
||||
let mm = tract_linalg::ops()
|
||||
.mmm(F32, F32, F32, Some(m), Some(k), Some(n))
|
||||
.unwrap();
|
||||
let pa =
|
||||
Tensor::uninitialized_aligned::<f32>(&[mm.a_pack().len(k, m)], mm.a_pack().alignment())
|
||||
.unwrap();
|
||||
let pb =
|
||||
Tensor::uninitialized_aligned::<f32>(&[mm.b_pack().len(k, n)], mm.b_pack().alignment())
|
||||
.unwrap();
|
||||
let mut c = Tensor::zero::<f32>(&[m, n]).unwrap();
|
||||
be.iter(move || {
|
||||
mm.run(
|
||||
m,
|
||||
n,
|
||||
&[
|
||||
FusedSpec::AddMatMul {
|
||||
a: mm.a_packed(F32.size_of(), k).wrap(&pa.view()),
|
||||
b: mm.b_packed(F32.size_of(), k).wrap(&pb.view()).unwrap(),
|
||||
k,
|
||||
},
|
||||
FusedSpec::Store(mm.c_view(0, 1).wrap(&c.view_mut())),
|
||||
],
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn mat_mul_prepacked(c: &mut Criterion, m: usize, k: usize, n: usize) {
|
||||
let mut group = c.benchmark_group("mat_mul_prepacked");
|
||||
group.bench_function("smmm", |be| mat_mul_smmm(be, &(m, k, n)));
|
||||
}
|
||||
|
||||
fn s64x288x21609(c: &mut Criterion) {
|
||||
mat_mul_prepacked(c, 64, 288, 21609)
|
||||
}
|
||||
|
||||
criterion::criterion_group!(benches, s64x288x21609);
|
||||
criterion::criterion_main!(benches);
|
||||
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
use criterion::*;
|
||||
|
||||
mod utils;
|
||||
use utils::*;
|
||||
|
||||
fn s16x60x8(c: &mut Criterion) {
|
||||
packed_packed(c, "wavenet", 32, 32, 8); // postproc
|
||||
packed_packed(c, "wavenet", 16, 60, 8);
|
||||
}
|
||||
|
||||
criterion_group!(benches, s16x60x8);
|
||||
criterion_main!(benches);
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user