Add native self-hosted instance connection to fluxer_desktop
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2024"
|
||||
name = "tract-linalg"
|
||||
version = "0.23.1"
|
||||
authors = ["Mathieu Poumeyrol <kali@zoy.org>"]
|
||||
build = "build.rs"
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference"
|
||||
readme = "README.md"
|
||||
keywords = [
|
||||
"TensorFlow",
|
||||
"NeuralNetworks",
|
||||
]
|
||||
categories = ["science"]
|
||||
license = "MIT OR Apache-2.0"
|
||||
repository = "https://github.com/snipsco/tract"
|
||||
resolver = "2"
|
||||
|
||||
[badges.maintenance]
|
||||
status = "actively-developed"
|
||||
|
||||
[features]
|
||||
apple-amx-ios = []
|
||||
complex = ["tract-data/complex"]
|
||||
default = []
|
||||
hwbench = ["rayon"]
|
||||
multithread-mm = ["rayon"]
|
||||
no_fp16 = []
|
||||
|
||||
[lib]
|
||||
name = "tract_linalg"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[test]]
|
||||
name = "virtual_im2col"
|
||||
path = "tests/virtual_im2col.rs"
|
||||
|
||||
[[bench]]
|
||||
name = "activations_avx512"
|
||||
path = "benches/activations_avx512.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "activations_avx512_f16"
|
||||
path = "benches/activations_avx512_f16.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "activations_avx512_fp16"
|
||||
path = "benches/activations_avx512_fp16.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "arm32neon"
|
||||
path = "benches/arm32neon.rs"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "arm64"
|
||||
path = "benches/arm64.rs"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "arm64simd"
|
||||
path = "benches/arm64simd.rs"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "avx512_zombies"
|
||||
path = "benches/avx512_zombies.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "erf"
|
||||
path = "benches/erf.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "gelu"
|
||||
path = "benches/gelu.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "hardswish"
|
||||
path = "benches/hardswish.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "intel"
|
||||
path = "benches/intel.rs"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "leaky_relu"
|
||||
path = "benches/leaky_relu.rs"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mat_vec"
|
||||
path = "benches/mat_vec.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_asr_am"
|
||||
path = "benches/mm_for_asr_am.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_inception"
|
||||
path = "benches/mm_for_inception.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_wavenet_hw"
|
||||
path = "benches/mm_for_wavenet_hw.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "qmmm_i8"
|
||||
path = "benches/qmmm_i8.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "rms_norm"
|
||||
path = "benches/rms_norm.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "sigmoid"
|
||||
path = "benches/sigmoid.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "silu"
|
||||
path = "benches/silu.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "softmax"
|
||||
path = "benches/softmax.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "virtual_im2col"
|
||||
path = "benches/virtual_im2col.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "vnni_i32"
|
||||
path = "benches/vnni_i32.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "wasm"
|
||||
path = "benches/wasm.rs"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "x86_64"
|
||||
path = "benches/x86_64.rs"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[dependencies.byteorder]
|
||||
version = "1.4.3"
|
||||
|
||||
[dependencies.derive-new]
|
||||
version = "0.7"
|
||||
|
||||
[dependencies.downcast-rs]
|
||||
version = "2.0"
|
||||
|
||||
[dependencies.dyn-clone]
|
||||
version = "1.0.4"
|
||||
|
||||
[dependencies.dyn-eq]
|
||||
version = "0.1"
|
||||
|
||||
[dependencies.dyn-hash]
|
||||
version = "1.0"
|
||||
|
||||
[dependencies.lazy_static]
|
||||
version = "1.5.0"
|
||||
|
||||
[dependencies.log]
|
||||
version = "0.4.14"
|
||||
|
||||
[dependencies.num-traits]
|
||||
version = "0.2.14"
|
||||
|
||||
[dependencies.pastey]
|
||||
version = "0.2"
|
||||
|
||||
[dependencies.rayon]
|
||||
version = "1.10"
|
||||
optional = true
|
||||
|
||||
[dependencies.scan_fmt]
|
||||
version = "0.2.6"
|
||||
|
||||
[dependencies.tract-data]
|
||||
version = "=0.23.1"
|
||||
|
||||
[dev-dependencies.core_affinity]
|
||||
version = "0.8.0"
|
||||
|
||||
[dev-dependencies.env_logger]
|
||||
version = "0.11"
|
||||
|
||||
[dev-dependencies.libc]
|
||||
version = "0.2.164"
|
||||
|
||||
[dev-dependencies.nu-ansi-term]
|
||||
version = "0.50"
|
||||
|
||||
[build-dependencies.cc]
|
||||
version = "1.0.69"
|
||||
|
||||
[build-dependencies.half]
|
||||
version = ">=2.4,<3.0"
|
||||
features = [
|
||||
"std",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[build-dependencies.minijinja]
|
||||
version = "2.19.0"
|
||||
|
||||
[build-dependencies.walkdir]
|
||||
version = "2.3.2"
|
||||
|
||||
[target.'cfg(not(target_family = "wasm"))'.dev-dependencies.criterion]
|
||||
version = "0.8"
|
||||
|
||||
[target.'cfg(not(target_family = "wasm"))'.dev-dependencies.proptest]
|
||||
version = "1.0.0"
|
||||
|
||||
[target.'cfg(target_family = "wasm")'.dev-dependencies.criterion]
|
||||
version = "0.8"
|
||||
features = [
|
||||
"plotters",
|
||||
"cargo_bench_support",
|
||||
]
|
||||
default-features = false
|
||||
|
||||
[target.'cfg(target_family = "wasm")'.dev-dependencies.proptest]
|
||||
version = "1.0.0"
|
||||
features = [
|
||||
"std",
|
||||
"bit-set",
|
||||
]
|
||||
default-features = false
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
[package]
|
||||
name = "tract-linalg"
|
||||
version = "0.23.1"
|
||||
license = "MIT OR Apache-2.0"
|
||||
authors = ["Mathieu Poumeyrol <kali@zoy.org>"]
|
||||
description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference"
|
||||
repository = "https://github.com/snipsco/tract"
|
||||
keywords = ["TensorFlow", "NeuralNetworks"]
|
||||
categories = ["science"]
|
||||
autobenches = false
|
||||
edition = "2024"
|
||||
|
||||
[badges]
|
||||
maintenance = { status = "actively-developed" }
|
||||
|
||||
[dependencies]
|
||||
byteorder.workspace = true
|
||||
derive-new.workspace = true
|
||||
downcast-rs.workspace = true
|
||||
dyn-clone.workspace = true
|
||||
dyn-eq.workspace = true
|
||||
dyn-hash.workspace = true
|
||||
lazy_static.workspace = true
|
||||
log.workspace = true
|
||||
num-traits.workspace = true
|
||||
pastey.workspace = true
|
||||
rayon = { workspace = true, optional = true }
|
||||
scan_fmt.workspace = true
|
||||
tract-data.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
cc.workspace = true
|
||||
half.workspace = true
|
||||
minijinja.workspace = true
|
||||
walkdir.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
env_logger.workspace = true
|
||||
libc.workspace = true
|
||||
nu-ansi-term.workspace = true
|
||||
core_affinity.workspace = true
|
||||
|
||||
[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
|
||||
criterion.workspace = true
|
||||
proptest.workspace = true
|
||||
|
||||
[target.'cfg(target_family = "wasm")'.dev-dependencies]
|
||||
# Wasm doesn't support the `rayon` feature of criterion
|
||||
criterion = { version = "0.8", default-features = false, features = ["plotters", "cargo_bench_support"] }
|
||||
# Wasm doesn't support the `fork` feature of proptest.
|
||||
proptest = { version = "1.0.0", default-features = false, features = ["std", "bit-set"] }
|
||||
|
||||
[features]
|
||||
# This feature is meant to accomodate very restrictive / legacy toolchains that do
|
||||
# have support for fp16 instructions, breaking tract compilation.
|
||||
# It is not meant to be used in other situations, where run-time detection is
|
||||
# preferred.
|
||||
no_fp16 = []
|
||||
apple-amx-ios = []
|
||||
default = [ ]
|
||||
multithread-mm = [ "rayon" ]
|
||||
complex = [ "tract-data/complex" ]
|
||||
hwbench = [ "rayon" ]
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "arm64"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mat_vec"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_wavenet_hw"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_inception"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_asr_am"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "qmmm_i8"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "hardswish"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "silu"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "gelu"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "sigmoid"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "softmax"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "activations_avx512_f16"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "erf"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "rms_norm"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "activations_avx512_fp16"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "arm64simd"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "arm32neon"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "virtual_im2col"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "x86_64"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "intel"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
bench = false
|
||||
name = "leaky_relu"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "avx512_zombies"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "activations_avx512"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "wasm"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "vnni_i32"
|
||||
harness = false
|
||||
@@ -0,0 +1,12 @@
|
||||
## License
|
||||
|
||||
Licensed under either of
|
||||
* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
|
||||
* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
|
||||
at your option.
|
||||
|
||||
### Contribution
|
||||
|
||||
Unless you explicitly state otherwise, any contribution intentionally submitted
|
||||
for inclusion in the work by you, as defined in the Apache-2.0 license, shall
|
||||
be dual licensed as above, without any additional terms or conditions.
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,23 @@
|
||||
Permission is hereby granted, free of charge, to any
|
||||
person obtaining a copy of this software and associated
|
||||
documentation files (the "Software"), to deal in the
|
||||
Software without restriction, including without
|
||||
limitation the rights to use, copy, modify, merge,
|
||||
publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software
|
||||
is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
The above copyright notice and this permission notice
|
||||
shall be included in all copies or substantial portions
|
||||
of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
||||
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
|
||||
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
||||
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
|
||||
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
# Multithreaded MMM benchmarks
|
||||
|
||||
Validation data for the `multithread-mm` rayon path with this PR's
|
||||
`chunked_dispatch_rayon` + `THREADING_PANEL_THRESHOLD` + `RayonGlobal`
|
||||
changes.
|
||||
|
||||
## Setup
|
||||
|
||||
- **Base**: tract `main` (commit `41b7b02`), with the merged WASM kernel
|
||||
kit (PRs `#2164` + `#2173`).
|
||||
- **Vanilla baseline**: same commit, MMM dispatch unchanged
|
||||
(1D `into_par_iter` over single panel axis).
|
||||
- **Patched**: this PR applied — chunked 2D dispatch, threshold, RayonGlobal.
|
||||
- **Both binaries built identically**: same compiler, same kernel kit, same
|
||||
`+atomics +bulk-memory +mutable-globals +simd128` target features for
|
||||
WASM.
|
||||
- **Driver**: Playwright headless, real browser engines. Median of 60
|
||||
iterations after 3-iter warmup.
|
||||
- **Output verification**: FNV-1a hash of result tensor. All 60 cells
|
||||
produce identical hash (`20ea4579c427f925` for DFN3,
|
||||
shape-deterministic for synthetic) — bit-equal output preserved.
|
||||
|
||||
## Synthetic dense matmul (the parallelism-bound case)
|
||||
|
||||
| Shape | Engine | Vanilla (1 thread) | Patched, 4 threads | Speedup |
|
||||
|---|---|---|---|---|
|
||||
| **1024×1024×1024** (transformer FFN) | Chromium | 55.4 ms | 16.6 ms | **3.34×** |
|
||||
| | WebKit | 79.8 ms | 25.6 ms | **3.12×** |
|
||||
| | Firefox | 1107 ms | 325 ms | **3.40×** |
|
||||
| **512×768×768** (BERT FFN) | Chromium | 15.7 ms | 5.1 ms | **3.07×** |
|
||||
| | WebKit | 22.6 ms | 6.8 ms | **3.30×** |
|
||||
| | Firefox | 311 ms | 91 ms | **3.42×** |
|
||||
| **256×256×128** | Chromium | 0.47 ms | 0.22 ms | **2.19×** |
|
||||
| | WebKit | 0.66 ms | 0.20 ms | **3.30×** |
|
||||
| | Firefox | 9.0 ms | 2.6 ms | **3.41×** |
|
||||
| 64×256×64 (DFN-like small) | Chromium | 0.07 ms | 0.07 ms | 1.0× (within noise) |
|
||||
| | WebKit | 0.08 ms | 0.04 ms | 2.00× |
|
||||
| | Firefox | 1.18 ms | 0.38 ms | **3.11×** |
|
||||
| 32×32×32 (tiny) | All | sub-ms | sub-ms | **1.0× (threshold gates)** |
|
||||
|
||||
The threshold correctly gates the smallest shape; threading kicks in once
|
||||
panel count clears the gate, and scales near-linearly with thread count
|
||||
on all three engines.
|
||||
|
||||
## Real model (DeepFilterNet 3, full streaming inference)
|
||||
|
||||
5-frame chunks at 48 kHz (50 ms of audio per iteration).
|
||||
|
||||
| Engine | Vanilla mono | Patched, 4 threads | RTF (vanilla → patched) | Speedup |
|
||||
|---|---|---|---|---|
|
||||
| Chromium | 3.31 ms | 3.17 ms | 0.066 → 0.063 | 1.04× (within noise) |
|
||||
| WebKit | 4.32 ms | 4.00 ms | 0.086 → 0.080 | 1.08× |
|
||||
| Firefox | 34.20 ms | 33.34 ms | 0.684 → 0.667 | 1.03× |
|
||||
|
||||
DFN3 is Amdahl-bound: only ~25% of runtime is in MMMs that clear the
|
||||
threshold; the rest is FFT, complex multiplication, and small RNN-internal
|
||||
matmuls that the threshold deliberately keeps single-threaded. The
|
||||
threshold's role here is to **prevent regression**, not deliver speedup.
|
||||
This is correct behavior — DFN3-class workloads should not pay rayon
|
||||
overhead on tiny ops.
|
||||
|
||||
## Native (macOS aarch64, generic kernels)
|
||||
|
||||
Spot-check on the rayon path before/after. Tract's existing rayon path
|
||||
already worked well on native; the change is mostly a refactor.
|
||||
|
||||
| Shape | Vanilla 1D | Patched 2D | Change |
|
||||
|---|---|---|---|
|
||||
| 256×256, 4 threads | 2.13 ms | 2.12 ms | net-neutral |
|
||||
| 512×512, 4 threads | 9.93 ms | 9.80 ms | +1% |
|
||||
| 64×256, 4 threads | 615 µs | 625 µs | −2% |
|
||||
|
||||
Within noise on common shapes. The 2D dispatch shows a latent benefit on
|
||||
shapes 1D parallelism handles poorly (e.g. m=8 n=2048, where 1D over m
|
||||
can only feed 2 threads); not yet measured directly on native but the
|
||||
dispatch math is the same on both targets.
|
||||
|
||||
## Determinism
|
||||
|
||||
Across all measured cells (synthetic + DFN3, 3 engines, 1/2/3/4 threads):
|
||||
|
||||
- **WASM**: 60 cells, all produce identical hash per `(shape, mode)` pair.
|
||||
- **Native**: existing tract proptests (3524 lib tests) pass with this
|
||||
PR's `multithread-mm` enabled.
|
||||
|
||||
## Tuning the threshold
|
||||
|
||||
The default `THREADING_PANEL_THRESHOLD` is `64` panels (m_panels ×
|
||||
n_panels). Adjust at runtime via:
|
||||
|
||||
```rust
|
||||
use tract_linalg::multithread::set_threading_panel_threshold;
|
||||
|
||||
set_threading_panel_threshold(0); // thread every size, no gate
|
||||
set_threading_panel_threshold(256); // gate harder — transformer-only
|
||||
set_threading_panel_threshold(64); // default
|
||||
```
|
||||
|
||||
Useful when profiling or specialising the build for a known workload class:
|
||||
|
||||
| Workload class | Suggested threshold |
|
||||
|---|---|
|
||||
| Streaming RNN / mobile vision (many small MMMs) | 64 (default) or higher |
|
||||
| Mid-size dense (BERT-class) | 32–64 |
|
||||
| Large dense only (transformer FFN, LLM) | 16 or lower |
|
||||
|
||||
The constant lives in `linalg/src/multithread.rs`; readers go through
|
||||
`current_threading_panel_threshold()` (`AtomicUsize::Relaxed`, no lock on
|
||||
the dispatch hot path).
|
||||
|
||||
## Reproduction
|
||||
|
||||
The harness uses [Vonage's libDF fork](https://github.com/czoli1976/DeepFilterNet)
|
||||
(branch `dfn3-wasm-opt-tract-022-kernel-kit`) migrated to tract main, with
|
||||
a `wasm-bindgen-rayon`-based threading bootstrap. Build:
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals,+simd128" \
|
||||
wasm-pack build --target web --release \
|
||||
--no-default-features --features wasm-mt -- \
|
||||
-Z build-std=std,panic_abort
|
||||
```
|
||||
|
||||
JS-side:
|
||||
|
||||
```javascript
|
||||
import init, { initThreadPool, df_set_thread_count } from './pkg/df.js';
|
||||
await init();
|
||||
await initThreadPool(navigator.hardwareConcurrency); // wasm-bindgen-rayon
|
||||
df_set_thread_count(4); // sets Executor::RayonGlobal in tract-linalg
|
||||
```
|
||||
|
||||
Without `Executor::RayonGlobal` (this PR), `df_set_thread_count` would
|
||||
need to construct an `Arc<rayon::ThreadPool>` — which fails on
|
||||
`wasm32-unknown-unknown` because `rayon::ThreadPoolBuilder::new().build()`
|
||||
internally calls `std::thread::spawn` (unsupported there). That's the
|
||||
crux of why this enabling change is needed in tract-linalg itself: any
|
||||
browser threading via wasm-bindgen-rayon would otherwise silently fall
|
||||
back to single-threaded.
|
||||
@@ -0,0 +1,27 @@
|
||||
# tract-linalg
|
||||
|
||||
linalg stands for "linear algebra". This is a misnamer. This crates contains
|
||||
low-level, architecture dependant optimisations used by tract-core.
|
||||
|
||||
# Functions
|
||||
|
||||
* MatMatMul: Extended matrix*matrix product:
|
||||
* inspired by Gotoblass and BLIS micro kernel approach
|
||||
* extended for convolution friendly addressing (fused img2col)
|
||||
* fused output pipeline (min, max, and a few more simple, fast ops)
|
||||
* f32*f32 -> f32 (à la sgemm)
|
||||
* i8*i8 -> i32 accumulator -> i32 storage
|
||||
* i8*i8 -> i32 accumulator -> i8 (with channel zeropoint and scale, and re-quantization pipeline)
|
||||
* f32 sigmoid and f32 tanh: at f32 precision, by a rationale function (no exponentiation)
|
||||
* byte-to-byte lookup table
|
||||
|
||||
# Implementations
|
||||
|
||||
| | generic fallback | armv6, vfp | armv7 neon | armv8 simd | x64 FMA
|
||||
|-------------------|--------------------|---------------|-------------------|-------------------|-----------------
|
||||
| MatMatMul f32 | | 4x4 | 8x4 | 8x8 | 16x6
|
||||
| MatMatMul i8->i8 | | | 8x4 | | 8x8
|
||||
| MatMatMul i8->i32 | | | | | 8x8
|
||||
| sigmoid f32 | | | 4n | 4n |
|
||||
| tanh f32 | | | 4n | 4n |
|
||||
| byte lookup | | | | |
|
||||
BIN
Binary file not shown.
+105
@@ -0,0 +1,105 @@
|
||||
# tract-linalg on `wasm32` — relaxed-simd FMA
|
||||
|
||||
The WASM MMM kernels (`wasm_f32_4x4`, `4x1`, `8x1`, `16x1`, `32x1`, `8x8`)
|
||||
and the WASM sigmoid/tanh activations all flip between two emit modes at
|
||||
compile time, gated on `cfg(target_feature = "relaxed-simd")`:
|
||||
|
||||
- **Without** `+relaxed-simd`: pure `f32x4_add(_, f32x4_mul(_, _))` (mul+add).
|
||||
Runs on any WASM runtime that supports `simd128`.
|
||||
- **With** `+relaxed-simd`: `f32x4_relaxed_madd(_, _, _)`. Fused, single-rounded
|
||||
multiply-add on hosts whose CPU has hardware FMA (all ARM64, x86_64 + FMA3).
|
||||
Universal browser/runtime support since 2023 (Chrome 114+, Firefox 120+,
|
||||
Safari 17+, wasmtime 16+).
|
||||
|
||||
The speedup of the relaxed path over the baseline is typically **1.40–1.55× at
|
||||
the kernel level** and **1.08–1.46× end-to-end** across vision CNNs,
|
||||
transformer attention and RNN audio models. Bit-pattern drift versus the
|
||||
mul+add path is bounded at one ulp (FMA single-rounding); within
|
||||
`Approximation::Close` (1e-4).
|
||||
|
||||
## Build flags
|
||||
|
||||
```sh
|
||||
# Baseline (any wasm32 runtime supporting simd128)
|
||||
RUSTFLAGS='-C target-feature=+simd128' \
|
||||
cargo build --release --target wasm32-wasip1 -p tract-linalg
|
||||
|
||||
# Relaxed (requires host support for relaxed-simd; ~1.40× faster on FMA-capable hosts)
|
||||
RUSTFLAGS='-C target-feature=+simd128,+relaxed-simd' \
|
||||
cargo build --release --target wasm32-wasip1 -p tract-linalg
|
||||
```
|
||||
|
||||
Same on `wasm32-unknown-unknown` if shipping for the browser.
|
||||
|
||||
## Why two binaries (and not in-process runtime dispatch)
|
||||
|
||||
WASM validates the entire module at instantiation, before any code runs.
|
||||
A binary containing `f32x4.relaxed_madd` fails to instantiate on hosts without
|
||||
relaxed-simd — `LinkError` / `CompileError`, not a runtime trap. So the
|
||||
x86/ARM pattern (one binary, both paths in source, runtime CPU detection picks
|
||||
at execution time) cannot be replicated in-binary on WASM: the FMA opcodes are
|
||||
either present (and host support is required) or absent.
|
||||
|
||||
Runtime dispatch happens one layer up — at the host runtime / consumer layer
|
||||
— by selecting the correct binary at module-load time.
|
||||
|
||||
## Consumer-side dispatch
|
||||
|
||||
### Browser / `WebAssembly.validate`
|
||||
|
||||
```js
|
||||
async function loadTract(baseUrl) {
|
||||
const candidate = await fetch(`${baseUrl}/tract-relaxed.wasm`);
|
||||
const bytes = await candidate.arrayBuffer();
|
||||
|
||||
const wantRelaxed = WebAssembly.validate(bytes, {
|
||||
builtins: ['relaxed_simd'],
|
||||
});
|
||||
|
||||
const url = wantRelaxed
|
||||
? `${baseUrl}/tract-relaxed.wasm`
|
||||
: `${baseUrl}/tract.wasm`;
|
||||
|
||||
const final = await fetch(url);
|
||||
return WebAssembly.instantiateStreaming(final);
|
||||
}
|
||||
```
|
||||
|
||||
Fallback for hosts without the `WebAssembly.validate(bytes, { ... })`
|
||||
options-arg: try-instantiate the relaxed binary, catch `LinkError` /
|
||||
`CompileError`, retry with the baseline.
|
||||
|
||||
### `wasmtime` (server / native)
|
||||
|
||||
```rust
|
||||
use wasmtime::{Config, Engine};
|
||||
|
||||
let mut config = Config::new();
|
||||
config.wasm_relaxed_simd(true); // gate on host-CPU detection if needed
|
||||
let engine = Engine::new(&config)?;
|
||||
|
||||
let bytes = std::fs::read(if relaxed_supported {
|
||||
"tract-relaxed.wasm"
|
||||
} else {
|
||||
"tract.wasm"
|
||||
})?;
|
||||
let module = wasmtime::Module::new(&engine, &bytes)?;
|
||||
```
|
||||
|
||||
`wasmtime::Engine`'s `wasm_relaxed_simd` configures the runtime; a separate
|
||||
`wasmtime::Module::validate()` call against the engine is the equivalent of
|
||||
the browser's `WebAssembly.validate` for picking which binary to load.
|
||||
|
||||
## Quality
|
||||
|
||||
The two binaries are **not bit-identical**. FMA's single-rounding produces
|
||||
≤1 ulp drift from explicit mul+add. Verified end-to-end on Inception v3 and
|
||||
DFN3 sub-models:
|
||||
|
||||
| model | output shape | baseline L2 | relaxed L2 |
|
||||
|--------------|--------------------|-------------:|-------------:|
|
||||
| Inception v3 | [1, 1001] | 6.477089e-2 | 6.477089e-2 |
|
||||
| DFN3 df_dec | [1, 100, 96, 10] | 1.080686e-2 | 1.080686e-2 |
|
||||
|
||||
L2 norms are bit-identical to 7 sig figs; per-element values diverge in the
|
||||
7th–8th decimal place. Within tract's `Approximation::Close` (1e-4).
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0]
|
||||
// q8[1]
|
||||
// q8[2]
|
||||
// q8[3]
|
||||
//
|
||||
// ....
|
||||
//
|
||||
// q15[0]
|
||||
// q15[1]
|
||||
// q15[2]
|
||||
// q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_32x1_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
pld [r1, #128]
|
||||
pld [r1, #192]
|
||||
pld [r1, #256]
|
||||
pld [r1, #320]
|
||||
pld [r1, #384]
|
||||
pld [r1, #448]
|
||||
pld [r1, #512]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r1, #32]
|
||||
vldr d5, [r1, #40]
|
||||
vldr d6, [r1, #48]
|
||||
vldr d7, [r1, #56]
|
||||
vldr d8, [r1, #64]
|
||||
vldr d9, [r1, #72]
|
||||
vldr d10, [r1, #80]
|
||||
vldr d11, [r1, #88]
|
||||
vldr s30, [r5]
|
||||
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
|
||||
vldr d0, [r1, #96]
|
||||
vldr d1, [r1, #104]
|
||||
vldr d2, [r1, #112]
|
||||
vldr d3, [r1, #120]
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
add r1, #128
|
||||
add r5, #4
|
||||
|
||||
{% elif core == "cortexa9" %}
|
||||
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4-d7}, [r1]!
|
||||
pld [r1, #512]
|
||||
pld [r1, #576]
|
||||
vld1.64 {d8-d11}, [r1]!
|
||||
vld1.f32 d15[0], [r5]!
|
||||
pld [r5, #64]
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
|
||||
vmla.f32 q12, q4, d15[0]
|
||||
vmla.f32 q13, q5, d15[0]
|
||||
|
||||
vmla.f32 q14, q0, d15[0]
|
||||
vmla.f32 q15, q1, d15[0]
|
||||
|
||||
{% else %}
|
||||
|
||||
vldmia r1!, { q0-q3 }
|
||||
vldmia r5!, { s30 }
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
vldmia r1!, { q0-q1 }
|
||||
|
||||
vmla.f32 q10, q2, d15[0]
|
||||
vmla.f32 q11, q3, d15[0]
|
||||
vldmia r1!, { q2-q3 }
|
||||
|
||||
vmla.f32 q12, q0, d15[0]
|
||||
vmla.f32 q13, q1, d15[0]
|
||||
|
||||
vmla.f32 q14, q2, d15[0]
|
||||
vmla.f32 q15, q3, d15[0]
|
||||
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
{% for reg in range(0, 16) %}
|
||||
vld1.f32 d{{reg}}[0], [ r3 ], r4
|
||||
vld1.f32 d{{reg}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
{% for reg in range(0, 8) %}
|
||||
vadd.f32 q{{ reg + 8 }}, q{{ reg + 8 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vld1.f32 d0[0], [ r4 ]
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q8, q4, d0[0]
|
||||
vmla.f32 q9, q5, d0[0]
|
||||
|
||||
vmla.f32 q10, q6, d0[0]
|
||||
vmla.f32 q11, q7, d0[0]
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.f32 q12, q4, d0[0]
|
||||
vmla.f32 q13, q5, d0[0]
|
||||
|
||||
vmla.f32 q14, q6, d0[0]
|
||||
vmla.f32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4 <- ptr, rsc
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
vst1.f64 {d16-d19}, [r3]!
|
||||
vst1.f64 {d20-d23}, [r3]!
|
||||
vst1.f64 {d24-d27}, [r3]!
|
||||
vst1.f64 {d28-d31}, [r3]!
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
vst1.f32 d{{reg}}[0], [r3], r4
|
||||
vst1.f32 d{{reg}}[1], [r3], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x1_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
pld [r1, #128]
|
||||
pld [r1, #192]
|
||||
pld [r1, #256]
|
||||
pld [r1, #320]
|
||||
pld [r1, #384]
|
||||
pld [r1, #448]
|
||||
pld [r1, #512]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
vldmia r1!, { q0-q1 }
|
||||
vldmia r5!, { s30 }
|
||||
|
||||
vmla.f32 q8, q0, d15[0]
|
||||
vmla.f32 q9, q1, d15[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 9 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.f32 d{{reg}}[0], [ r3 ], r4
|
||||
vld1.f32 d{{reg}}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
{% for reg in range(0, 2) %}
|
||||
vadd.f32 q{{ reg + 8 }}, q{{ reg + 8 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vld1.f32 d0[0], [ r4 ]
|
||||
vldmia r3!, { q4-q5 }
|
||||
|
||||
vmla.f32 q8, q4, d0[0]
|
||||
vmla.f32 q9, q5, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4 <- ptr, rsc
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
vst1.f64 {d16-d19}, [r3]!
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
{% for reg in range(16, 20) %}
|
||||
vst1.f32 d{{reg}}[0], [r3], r4
|
||||
vst1.f32 d{{reg}}[1], [r3], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating q0, q1 with q2, q3
|
||||
// packed B buffering (2x4 values): alternating q4 with q5
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x4_{{core}}_{{suffix}}:
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
pld [r5] // packed B ptr
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
{% elif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, { q0, q1}
|
||||
vldmia r5!, { q2 }
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q8, q0, d4[0]
|
||||
vmla.f32 q9, q1, d4[0]
|
||||
|
||||
vmla.f32 q10, q0, d4[1]
|
||||
vmla.f32 q11, q1, d4[1]
|
||||
|
||||
vmla.f32 q12, q0, d5[0]
|
||||
vmla.f32 q13, q1, d5[0]
|
||||
|
||||
vmla.f32 q14, q0, d5[1]
|
||||
vmla.f32 q15, q1, d5[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #16
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
{% for col in range(0, 4) %}
|
||||
mov r2, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{ col * 4 + reg + 16 }}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.f32 q8, q0, d8[0]
|
||||
vmla.f32 q9, q1, d8[0]
|
||||
|
||||
vmla.f32 q10, q0, d8[1]
|
||||
vmla.f32 q11, q1, d8[1]
|
||||
|
||||
vmla.f32 q12, q0, d9[0]
|
||||
vmla.f32 q13, q1, d9[0]
|
||||
|
||||
vmla.f32 q14, q0, d9[1]
|
||||
vmla.f32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3,r4,r5 are c,rsc,csc
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vst1.f32 d{{ col * 4 + reg + 16 }}[0], [ r8 ], r4
|
||||
vst1.f32 d{{ col * 4 + reg + 16 }}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}
|
||||
.type armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_f32_8x6_{{core}}_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
vldr d0, [r1]
|
||||
vldr d1, [r1, #8]
|
||||
vldr d2, [r1, #16]
|
||||
vldr d3, [r1, #24]
|
||||
vldr d4, [r5]
|
||||
vldr d5, [r5, #8]
|
||||
vldr d6, [r5, #16]
|
||||
{% elif core == "cortexa9" %}
|
||||
vld1.64 {d0-d3}, [r1]!
|
||||
vld1.64 {d4, d5, d6}, [r5]!
|
||||
{% else %}
|
||||
vldmia r1!, {q0-q1}
|
||||
vldmia r5!, {d4-d6}
|
||||
{% endif %}
|
||||
|
||||
{% if core != "generic" %}
|
||||
pld [r1, #512]
|
||||
pld [r5, #512]
|
||||
{% endif %}
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
{% if core == "cortexa7" %}
|
||||
add r1, #32
|
||||
add r5, #24
|
||||
{% endif %}
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 4 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 4 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 4 %}{% set to = 15 %}{% include "armv7neon_mmm_f32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
{% for col in range(0, 6) %}
|
||||
mov r2, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.f32 d0[0], [ r2 ], r4
|
||||
vld1.f32 d0[1], [ r2 ], r4
|
||||
vadd.f32 d{{ col * 4 + reg + 8 }}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { d4, d5, d6 }
|
||||
|
||||
vmla.f32 q4, q0, d4[0]
|
||||
vmla.f32 q5, q1, d4[0]
|
||||
|
||||
vmla.f32 q6, q0, d4[1]
|
||||
vmla.f32 q7, q1, d4[1]
|
||||
|
||||
vmla.f32 q8, q0, d5[0]
|
||||
vmla.f32 q9, q1, d5[0]
|
||||
|
||||
vmla.f32 q10, q0, d5[1]
|
||||
vmla.f32 q11, q1, d5[1]
|
||||
|
||||
vmla.f32 q12, q0, d6[0]
|
||||
vmla.f32 q13, q1, d6[0]
|
||||
|
||||
vmla.f32 q14, q0, d6[1]
|
||||
vmla.f32 q15, q1, d6[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
// r3, r4, r5 <- ptr, rsc, csc
|
||||
|
||||
cmp r4, #4
|
||||
bne .store_generic
|
||||
|
||||
{% for col in range(0, 6) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vst1.64 d{{ col * 4 + 8 + reg }}, [ r8 ]!
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
{% for col in range(0, 6) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vst1.f32 d{{ col * 4 + reg + 8 }}[0], [ r8 ], r4
|
||||
vst1.f32 d{{ col * 4 + reg + 8 }}[1], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 5 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q4-q7 }
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col("per_col_min", "vmin.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_max", "vmax.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_mul", "vmul.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_add", "vadd.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub", "vsub.f32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub_flipped", "vsub.f32", mr, from, to, flipped=true) }}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row("per_row_min", "vmin.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_max", "vmax.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_mul", "vmul.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_add", "vadd.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub", "vsub.f32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub_flipped", "vsub.f32", mr, from, to, flipped=true) }}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar("scalar_min", "vmin.f32", from, to) }}
|
||||
{{ scalar("scalar_max", "vmax.f32", from, to) }}
|
||||
{{ scalar("scalar_mul", "vmul.f32", from, to) }}
|
||||
{{ scalar("scalar_add", "vadd.f32", from, to) }}
|
||||
{{ scalar("scalar_sub", "vsub.f32", from, to) }}
|
||||
{{ scalar("scalar_sub_flipped", "vsub.f32", from, to, flipped=true) }}
|
||||
|
||||
.leaky_relu:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% for reg in range(from, to + 1) %}
|
||||
vmul.f32 q2, q{{reg}}, q0
|
||||
vcgt.f32 q1, q{{reg}}, 0
|
||||
vbsl q1, q{{reg}}, q2
|
||||
vmov q{{reg}}, q1
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
.q_scale:
|
||||
b .unsupported
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: q8..q16
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_32x1_{{suffix}}
|
||||
.type armv7neon_mmm_i32_32x1_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_32x1_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
// r3 r4 r5 r6
|
||||
// k a b packing
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r7]
|
||||
|
||||
cmp r6, #1
|
||||
beq .packed_packed_i8i8
|
||||
|
||||
.packed_packed:
|
||||
|
||||
.packed_packed_loop_1:
|
||||
vldmia r1!, { q4-q7 }
|
||||
|
||||
vld1.32 { d0[0] }, [ r5 ]!
|
||||
|
||||
vmla.s32 q8, q4, d0[0]
|
||||
|
||||
vldmia r1!, { q1-q4 }
|
||||
|
||||
vmla.s32 q9, q5, d0[0]
|
||||
vmla.s32 q10, q6, d0[0]
|
||||
vmla.s32 q11, q7, d0[0]
|
||||
|
||||
vmla.s32 q12, q1, d0[0]
|
||||
vmla.s32 q13, q2, d0[0]
|
||||
|
||||
vmla.s32 q14, q3, d0[0]
|
||||
vmla.s32 q15, q4, d0[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_i8i8:
|
||||
|
||||
.packed_packed_loop_i8i8_1:
|
||||
vldmia r1!, { q4-q5 }
|
||||
|
||||
vld1.8 { d0[0] }, [ r5 ]!
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmovl.s8 q1, d8
|
||||
vmlal.s16 q8, d2, d0[0]
|
||||
vmlal.s16 q9, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d9
|
||||
vmlal.s16 q10, d2, d0[0]
|
||||
vmlal.s16 q11, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d10
|
||||
vmlal.s16 q12, d2, d0[0]
|
||||
vmlal.s16 q13, d3, d0[0]
|
||||
|
||||
vmovl.s8 q1, d11
|
||||
vmlal.s16 q14, d2, d0[0]
|
||||
vmlal.s16 q15, d3, d0[0]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_i8i8_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
vld1.s8 d0[0], [ r3 ], r4
|
||||
vld1.s8 d0[1], [ r3 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
{% for reg in range(16, 32) %}
|
||||
vld1.s32 d0[0], [ r3 ], r4
|
||||
vld1.s32 d0[1], [ r3 ], r4
|
||||
vadd.i32 d{{reg}}, d0
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldm r4, { s0 }
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.s32 q8, q4, d0[0]
|
||||
vmla.s32 q9, q5, d0[0]
|
||||
|
||||
vmla.s32 q10, q6, d0[0]
|
||||
vmla.s32 q11, q7, d0[0]
|
||||
|
||||
vldmia r3!, { q4-q7 }
|
||||
|
||||
vmla.s32 q12, q4, d0[0]
|
||||
vmla.s32 q13, q5, d0[0]
|
||||
|
||||
vmla.s32 q14, q6, d0[0]
|
||||
vmla.s32 q15, q7, d0[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.j2" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in range(8, 16) %}
|
||||
vmovn.s32 d{{ reg * 2 }}, q{{reg}}
|
||||
vmovn.s16 d{{ reg * 2 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for reg in range(8, 16) %}
|
||||
{% set d = reg * 2 %}
|
||||
vst1.s8 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r3 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for reg in range(8, 16) %}
|
||||
{% set d = reg * 2 %}
|
||||
vst1.s32 d{{d}}[0], [ r3 ], r4
|
||||
vst1.s32 d{{d}}[1], [ r3 ], r4
|
||||
vst1.s32 d{{ d + 1 }}[0], [ r3 ], r4
|
||||
vst1.s32 d{{ d + 1 }}[1], [ r3 ], r4
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs
|
||||
//
|
||||
// q8[0] q10[0] q12[0] q14[0]
|
||||
// q8[1] q10[1] q12[1] q14[1]
|
||||
// q8[2] q10[2] q12[2] q14[2]
|
||||
// q8[3] q10[3] q12[3] q14[3]
|
||||
//
|
||||
// q9[0] q11[0] q13[0] q15[0]
|
||||
// q9[1] q11[1] q13[1] q15[1]
|
||||
// q9[2] q11[2] q13[2] q15[2]
|
||||
// q9[3] q11[3] q13[3] q15[3]
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_mmm_i32_8x4_{{suffix}}
|
||||
.type armv7neon_mmm_i32_8x4_{{suffix}}, %function
|
||||
|
||||
armv7neon_mmm_i32_8x4_{{suffix}}:
|
||||
|
||||
pld [r0]
|
||||
push { r4-r12 }
|
||||
vpush { q4-q7 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
// r3 r4 r5 r6
|
||||
// k a b packing
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
cmp r6, #1
|
||||
beq .packed_packed_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
vldmia r1!, { q0, q1 }
|
||||
vldmia r5!, { q2 }
|
||||
|
||||
vmla.s32 q8, q0, d4[0]
|
||||
vmla.s32 q9, q1, d4[0]
|
||||
|
||||
vmla.s32 q10, q0, d4[1]
|
||||
vmla.s32 q11, q1, d4[1]
|
||||
|
||||
vmla.s32 q12, q0, d5[0]
|
||||
vmla.s32 q13, q1, d5[0]
|
||||
|
||||
vmla.s32 q14, q0, d5[1]
|
||||
vmla.s32 q15, q1, d5[1]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_i8i8:
|
||||
pld [r5] // packed B ptr
|
||||
|
||||
cmp r3, #4
|
||||
blt .packed_packed_loop_i8i8_1
|
||||
|
||||
.packed_packed_loop_i8i8_4:
|
||||
pld [r1, #64]
|
||||
pld [r5, #64]
|
||||
|
||||
// q2: d4 -> d4,d5 A even cols (from r1)
|
||||
// q3: d6 -> d6,d7 A odd cols (from r1)
|
||||
// q0: s0 -> d0 : B even lines (from r5)
|
||||
// q1: s4 -> d2 : B odd lines (from r5)
|
||||
|
||||
// 0
|
||||
vldmia r1!, { d4 }
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 1
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vldmia r1!, { d4 }
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
vldmia r5!, { s0 }
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
// 2
|
||||
vmovl.s8 q2, d4
|
||||
vmovl.s8 q0, d0
|
||||
|
||||
vmlal.s16 q8, d4, d0[0]
|
||||
vmlal.s16 q9, d5, d0[0]
|
||||
|
||||
vldmia r1!, { d6 }
|
||||
|
||||
vmlal.s16 q10, d4, d0[1]
|
||||
vmlal.s16 q11, d5, d0[1]
|
||||
|
||||
vldmia r5!, { s4 }
|
||||
|
||||
vmlal.s16 q12, d4, d0[2]
|
||||
vmlal.s16 q13, d5, d0[2]
|
||||
|
||||
vmlal.s16 q14, d4, d0[3]
|
||||
vmlal.s16 q15, d5, d0[3]
|
||||
|
||||
// 3
|
||||
vmovl.s8 q3, d6
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d6, d2[0]
|
||||
vmlal.s16 q9, d7, d2[0]
|
||||
|
||||
vmlal.s16 q10, d6, d2[1]
|
||||
vmlal.s16 q11, d7, d2[1]
|
||||
|
||||
vmlal.s16 q12, d6, d2[2]
|
||||
vmlal.s16 q13, d7, d2[2]
|
||||
|
||||
vmlal.s16 q14, d6, d2[3]
|
||||
vmlal.s16 q15, d7, d2[3]
|
||||
|
||||
sub r3, r3, #4
|
||||
cmp r3, #4
|
||||
bge .packed_packed_loop_i8i8_4
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_i8i8_1:
|
||||
|
||||
vldmia r1!, { s0, s1 }
|
||||
vmovl.s8 q0, d0
|
||||
vldmia r5!, { s4 }
|
||||
vmovl.s8 q1, d2
|
||||
|
||||
vmlal.s16 q8, d0, d2[0]
|
||||
vmlal.s16 q9, d1, d2[0]
|
||||
|
||||
vmlal.s16 q10, d0, d2[1]
|
||||
vmlal.s16 q11, d1, d2[1]
|
||||
|
||||
vmlal.s16 q12, d0, d2[2]
|
||||
vmlal.s16 q13, d1, d2[2]
|
||||
|
||||
vmlal.s16 q14, d0, d2[3]
|
||||
vmlal.s16 q15, d1, d2[3]
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_i8i8_1
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 8 %}{% set to = 15 %}{% include "armv7neon_mmm_i32_per_cols.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .non_linear_addc_i32
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.s8 d0[0], [ r8 ], r4
|
||||
vld1.s8 d0[1], [ r8 ], r4
|
||||
vmovl.s8 q0, d0
|
||||
vmovl.s16 q0, d0
|
||||
vadd.i32 d{{ col * 4 + reg + 16 }}, d0
|
||||
{% endfor %}
|
||||
add r3, r3, r5
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.non_linear_addc_i32:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vld1.s32 d0[0], [ r8 ], r4
|
||||
vld1.s32 d0[1], [ r8 ], r4
|
||||
vadd.i32 d{{ col * 4 + reg + 16 }}, d0
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { q0, q1 }
|
||||
vldmia r4!, { q4 }
|
||||
|
||||
vmla.s32 q8, q0, d8[0]
|
||||
vmla.s32 q9, q1, d8[0]
|
||||
|
||||
vmla.s32 q10, q0, d8[1]
|
||||
vmla.s32 q11, q1, d8[1]
|
||||
|
||||
vmla.s32 q12, q0, d9[0]
|
||||
vmla.s32 q13, q1, d9[0]
|
||||
|
||||
vmla.s32 q14, q0, d9[1]
|
||||
vmla.s32 q15, q1, d9[1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "armv7neon_mmm_i32_scale_q8_q15.j2" %}
|
||||
|
||||
.store:
|
||||
// r3, r4, r5, r6 <- ptr, rsc, csc, size
|
||||
cmp r6, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in range(8, 16) %}
|
||||
vmovn.s32 d{{ reg * 2 }}, q{{reg}}
|
||||
vmovn.s16 d{{ reg * 2 }}, q{{reg}}
|
||||
{% endfor %}
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 2) %}
|
||||
{% set d = (col * 2 + reg) * 2 + 16 %}
|
||||
vst1.s8 d{{d}}[0], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[1], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[2], [ r8 ], r4
|
||||
vst1.s8 d{{d}}[3], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 2) %}
|
||||
vst1.s32 d{{ col * 4 + reg + 16 }}[{{lane}}], [ r8 ], r4
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { q8-q15 }
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col("per_col_min", "vmin.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_max", "vmax.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_mul", "vmul.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_add", "vadd.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub", "vsub.s32", mr, from, to) }}
|
||||
{{ per_col("per_col_sub_flipped", "vsub.s32", mr, from, to, flipped=true) }}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row("per_row_min", "vmin.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_max", "vmax.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_mul", "vmul.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_add", "vadd.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub", "vsub.s32", mr, from, to) }}
|
||||
{{ per_row("per_row_sub_flipped", "vsub.s32", mr, from, to, flipped=true) }}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
// vim: ft=arm
|
||||
{% from "armv7neon_mmm_q_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar("scalar_min", "vmin.s32", from, to) }}
|
||||
{{ scalar("scalar_max", "vmax.s32", from, to) }}
|
||||
{{ scalar("scalar_mul", "vmul.s32", from, to) }}
|
||||
{{ scalar("scalar_add", "vadd.s32", from, to) }}
|
||||
{{ scalar("scalar_sub", "vsub.s32", from, to) }}
|
||||
{{ scalar("scalar_sub_flipped", "vsub.s32", from, to, flipped=true) }}
|
||||
|
||||
.leaky_relu:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% for reg in range(from, to + 1) %}
|
||||
vmul.s32 q2, q{{reg}}, q0
|
||||
vcgt.s32 q1, q{{reg}}, 0
|
||||
vbsl q1, q{{reg}}, q2
|
||||
vmov q{{reg}}, q1
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.q_scale:
|
||||
ldm r0, { r4, r5, r6, r7 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q0, r7 // q0 <- multiplier
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
vmovl.s32 q1, d2
|
||||
|
||||
add r5, #32
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- -(shift + 32)
|
||||
vmovl.s32 q2, d4
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_scale_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_scale_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_scale_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_scale_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_scale_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_scale_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_scale_rounding_zero:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_away:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_minus_inf:
|
||||
{% for q in range(8, 16) %}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vsub.s64 q5, q1
|
||||
vsub.s64 q6, q1
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_plus_inf:
|
||||
{% for q in range(8, 16) %}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_even:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q3, q3, q1
|
||||
vsub.s64 q4, q4, q1
|
||||
vadd.s64 q5, q3
|
||||
vadd.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_odd:
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q7, q{{q}}, #0
|
||||
vabs.s32 q{{q}}, q{{q}}
|
||||
vqdmull.s32 q5, d{{ q * 2 }}, d0[0]
|
||||
vqdmull.s32 q6, d{{ q * 2 + 1 }}, d0[0]
|
||||
vqshl.s64 q3, q5, q2
|
||||
vqshl.s64 q4, q6, q2
|
||||
vand q3, q3, q1
|
||||
vand q4, q4, q1
|
||||
vsub.s64 q5, q3
|
||||
vsub.s64 q6, q4
|
||||
vqrshl.s64 q5, q2
|
||||
vqrshl.s64 q6, q2
|
||||
vmovn.s64 d{{ q * 2 }}, q5
|
||||
vmovn.s64 d{{ q * 2 + 1 }}, q6
|
||||
vneg.s32 q5, q{{q}}
|
||||
vbit.s32 q{{q}}, q5, q7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
ldm r0, { r4, r5 } // fixme params are already loaded by disp.
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
{% for q in range(8, 16) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Shift
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr:
|
||||
ldm r0, { r4, r5, r6 } // fixme params are already loaded by disp.
|
||||
|
||||
mov r3, #1
|
||||
vdup.s32 q1, r3 // q1 <- ones
|
||||
|
||||
neg r5, r5
|
||||
vdup.s32 q2, r5 // q2 <- shift
|
||||
|
||||
cmp r6, #1
|
||||
beq .q_shr_rounding_zero
|
||||
cmp r6, #2
|
||||
beq .q_shr_rounding_away
|
||||
cmp r6, #3
|
||||
beq .q_shr_rounding_minus_inf
|
||||
cmp r6, #4
|
||||
beq .q_shr_rounding_plus_inf
|
||||
cmp r6, #5
|
||||
beq .q_shr_rounding_even
|
||||
cmp r6, #6
|
||||
beq .q_shr_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_shr_rounding_zero:
|
||||
// return signum(x) * ((abs(x) - 1) >>r shift )
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vsub.s32 q{{q}}, q1 // Substract 1 to abs(x)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -((abs(x) - 1) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_away:
|
||||
// return signum(x) * (abs(x) >>r shift )
|
||||
{% for q in range(8, 16) %}
|
||||
vclt.s32 q3, q{{q}}, #0 // Store the sign of the value
|
||||
vabs.s32 q{{q}}, q{{q}} // Compute their abs
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q4, q{{q}} // Compute -(abs(x) >>r shift )
|
||||
vbit.s32 q{{q}}, q4, q3 // Restore sign of x with bit mask
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_minus_inf:
|
||||
// return -(-x >>r shift)
|
||||
{% for q in range(8, 16) %}
|
||||
vneg.s32 q3, q{{q}} // Compute -x
|
||||
vqrshl.s32 q3, q2 // Rounding shift (0.5 -> 1)
|
||||
vneg.s32 q{{q}}, q3 // Compute -(-x >>r shift)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_plus_inf:
|
||||
// return x >>r shift
|
||||
{% for q in range(8, 16) %}
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_even:
|
||||
// If (x >> shift) is odd -> (x - 0) >>r shift
|
||||
// If (x >> shift) is even -> (x - 1) >>r shift
|
||||
{% for q in range(8, 16) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x is odd
|
||||
vsub.s32 q5, q4, q1 // If (x >> shift) is odd 0 else -1
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If (x >> shift) is odd (x - 0) else (x - 1)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_odd:
|
||||
// If (x >> shift) is even -> (x - 0) >>r shift
|
||||
// If (x >> shift) is odd -> (x - 1) >>r shift
|
||||
{% for q in range(8, 16) %}
|
||||
vqshl.s32 q3, q{{q}}, q2 // Truncate shift (0.5 -> 0)
|
||||
vand.s32 q4, q3, q1 // Store if x >> shift is odd
|
||||
vneg.s32 q5, q4 // If x is odd -1 else 0
|
||||
vadd.s32 q{{q}}, q{{q}}, q5 // If x is odd (x - 1) else (x - 0)
|
||||
vqrshl.s32 q{{q}}, q2 // Rounding shift (0.5 -> 1)
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
// Jinja2 macros for armv7neon scalar / per-row / per-col operations.
|
||||
// Shared by both f32 and i32 kernel variants.
|
||||
|
||||
{% macro scalar(label, op, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
vmov s0, r3
|
||||
vdup.32 q0, d0[0]
|
||||
{% if flipped %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} q{{reg}}, q{{reg}}, q0
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} q{{reg}}, q0, q{{reg}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_row(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set mr_over_4_min_1 = mr // 4 - 1 %}
|
||||
|
||||
{% for reg in range(0, mr_over_4_min_1 + 1) %}
|
||||
vldmia r3!, { q{{reg}} }
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} q{{acc}}, q{{acc}}, q{{other}}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} q{{acc}}, q{{other}}, q{{acc}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_col(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set mr_over_4_min_1 = mr // 4 - 1 %}
|
||||
|
||||
{% set cols = (to + 1 - from) // mr_over_4 %}
|
||||
{% set cols_min_1 = (to + 1 - from) // mr_over_4 - 1 %}
|
||||
|
||||
{% if cols == 1 %}
|
||||
vld1.f32 d0[0], [ r3 ]
|
||||
{% else %}
|
||||
{% set cols_over_2_minus_1 = cols // 2 - 1 %}
|
||||
{% for c in range(0, cols_over_2_minus_1 + 1) %}
|
||||
vldmia r3!, { d{{c}} }
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% for right in range(0, cols_min_1 + 1) %}
|
||||
vdup.f32 q3, d{{ right // 2 }}[{{ right % 2 }}]
|
||||
{% for down in range(0, mr_over_4_min_1 + 1) %}
|
||||
{% set acc = mr_over_4 * right + from + down %}
|
||||
{% if flipped %}
|
||||
{{op}} q{{acc}}, q{{acc}}, q3
|
||||
{% else %}
|
||||
{{op}} q{{acc}}, q3, q{{acc}}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_prefetch_{{suffix}}
|
||||
.type armv7neon_prefetch_{{suffix}}, %function
|
||||
|
||||
armv7neon_prefetch_{{suffix}}:
|
||||
loop:
|
||||
pld [r0]
|
||||
pld [r0, #32]
|
||||
pld [r0, #64]
|
||||
pld [r0, #96]
|
||||
pld [r0, #128]
|
||||
pld [r0, #160]
|
||||
pld [r0, #192]
|
||||
pld [r0, #224]
|
||||
add r0, r0, #256
|
||||
cmp r0, r1
|
||||
blt loop
|
||||
|
||||
bx lr
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_sigmoid_f32_4n_{{suffix}}
|
||||
.type armv7neon_sigmoid_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_sigmoid_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vdup.32 q11, d6[1]
|
||||
vdup.32 q12, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
vmla.f32 q11, q5, q8
|
||||
vmla.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vdup.32 q10, d6[1]
|
||||
vmla.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -18.6 // low
|
||||
.float 18.6 // high
|
||||
.float -4.433153405e-18 // alpha_13
|
||||
.float 1.169974371e-14
|
||||
|
||||
.float -1.875289645e-11
|
||||
.float 4.257889523e-8
|
||||
.float 0.00004811817576
|
||||
.float 0.008163842030
|
||||
|
||||
.float 0.2499999971
|
||||
.float 3.922935744e-6 // beta_6
|
||||
.float 0.001524872358
|
||||
.float 0.1159886749
|
||||
|
||||
.float 1.0
|
||||
.float 0.5 //
|
||||
.float 0.0 // padding
|
||||
.float 0.0
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armv7neon_tanh_f32_4n_{{suffix}}
|
||||
.type armv7neon_tanh_f32_4n_{{suffix}}, %function
|
||||
|
||||
/*
|
||||
s16–s31 (d8–d15, q4–q7) must be preserved
|
||||
s0–s15 (d0–d7, q0–q3) and d16–d31 (q8–q15) do not need to be preserved
|
||||
*/
|
||||
|
||||
armv7neon_tanh_f32_4n_{{suffix}}:
|
||||
cmp r1, #0
|
||||
blxeq lr
|
||||
|
||||
vpush { q4-q7 }
|
||||
|
||||
adr r2, .coeffs_num
|
||||
vldmia r2!, { s0-s13 }
|
||||
|
||||
// q4 -> q4,5,6
|
||||
// q5 -> q7,8,9
|
||||
// q6 -> q10,11,12
|
||||
// q7 -> q13,14,15
|
||||
|
||||
cmp r1, #12
|
||||
blt .loop
|
||||
|
||||
.loop_3:
|
||||
vldmia r0, { q4, q5, q6 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vmax.f32 q5, q15
|
||||
vmax.f32 q6, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
vmin.f32 q5, q15
|
||||
vmin.f32 q6, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
vmul.f32 q8, q5, q5
|
||||
vmul.f32 q9, q6, q6
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q11, d1[0]
|
||||
vdup.32 q12, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vdup.32 q14, d1[1]
|
||||
vdup.32 q15, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d2[0]
|
||||
vdup.32 q11, d2[0]
|
||||
vdup.32 q12, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d2[1]
|
||||
vdup.32 q14, d2[1]
|
||||
vdup.32 q15, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d3[0]
|
||||
vdup.32 q11, d3[0]
|
||||
vdup.32 q12, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d3[1]
|
||||
vdup.32 q14, d3[1]
|
||||
vdup.32 q15, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d4[0]
|
||||
vdup.32 q11, d4[0]
|
||||
vdup.32 q12, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
vmul.f32 q5, q5, q11
|
||||
vmul.f32 q6, q6, q12
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q11, d4[1]
|
||||
vdup.32 q12, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vdup.32 q14, d5[0]
|
||||
vdup.32 q15, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
vdup.32 q10, d5[1]
|
||||
vdup.32 q11, d5[1]
|
||||
vdup.32 q12, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmla.f32 q11, q14, q8
|
||||
vmla.f32 q12, q15, q9
|
||||
vdup.32 q13, d6[0]
|
||||
vdup.32 q14, d6[0]
|
||||
vdup.32 q15, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
vmla.f32 q14, q8, q11
|
||||
vmla.f32 q15, q9, q12
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecpe.f32 q8, q14
|
||||
vrecpe.f32 q9, q15
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
vrecps.f32 q10, q7, q13
|
||||
vrecps.f32 q11, q8, q14
|
||||
vrecps.f32 q12, q9, q15
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
vmul.f32 q8, q8, q11
|
||||
vmul.f32 q9, q9, q12
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
vmul.f32 q11, q5, q8
|
||||
vmul.f32 q12, q6, q9
|
||||
|
||||
vstmia r0!, { q10, q11, q12 }
|
||||
|
||||
subs r1, #12
|
||||
cmp r1, #12
|
||||
bge .loop_3
|
||||
|
||||
cmp r1, #0;
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
vldmia r0, { q4 } // q4 <- x
|
||||
|
||||
vdup.32 q15, d0[0]
|
||||
vmax.f32 q4, q15
|
||||
vdup.32 q15, d0[1]
|
||||
vmin.f32 q4, q15
|
||||
|
||||
vmul.f32 q7, q4, q4 // q7 <- x2
|
||||
|
||||
vdup.32 q10, d1[0]
|
||||
vdup.32 q13, d1[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d2[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d2[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d3[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d3[1]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d4[0]
|
||||
vmla.f32 q10, q13, q7
|
||||
vmul.f32 q4, q4, q10 // q4 <- numerator
|
||||
|
||||
vdup.32 q10, d4[1]
|
||||
vdup.32 q13, d5[0]
|
||||
vmla.f32 q13, q7, q10
|
||||
vdup.32 q10, d5[1]
|
||||
vmla.f32 q10, q13, q7
|
||||
vdup.32 q13, d6[0]
|
||||
vmla.f32 q13, q7, q10 // q13 <- denum
|
||||
|
||||
vrecpe.f32 q7, q13
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10
|
||||
vrecps.f32 q10, q7, q13
|
||||
vmul.f32 q7, q7, q10 // q7 <- 1/q13
|
||||
|
||||
vmul.f32 q10, q4, q7
|
||||
|
||||
vstmia r0!, { q10 }
|
||||
|
||||
subs r1, #4;
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
vpop { q4-q7 }
|
||||
bx lr
|
||||
|
||||
.coeffs_num:
|
||||
.float -8.9 // low
|
||||
.float 8.9 // high
|
||||
.float -8.488492677e-14 // alpha_13
|
||||
.float 5.277853000e-11
|
||||
|
||||
.float -2.022500419e-8
|
||||
.float 0.00001115424833
|
||||
.float 0.003103950131
|
||||
.float 0.1308400453
|
||||
|
||||
.float 0.9999999934
|
||||
.float 0.0002546136580 // beta_6
|
||||
.float 0.02449515379
|
||||
.float 0.4641733162
|
||||
|
||||
.float 1.0
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
.float 0 // padding
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
|
||||
.non_linear_loop_entry:
|
||||
sub r0, #20
|
||||
|
||||
.non_linear_loop:
|
||||
add r0, #20
|
||||
ldm r0, { r2, r3, r4, r5, r6 }
|
||||
|
||||
cmp r2, #{{ jump_table | length }}
|
||||
movgt r2, #{{ jump_table | length }}
|
||||
cmp r2, #0
|
||||
movlt r2, #{{ jump_table | length }}
|
||||
|
||||
add pc, pc, r2, LSL#2
|
||||
nop // pc in Rn above is start of the add instruction + 8, hence a nop is needed
|
||||
// This is A32 asm, for T32/Thump2 use nop.w and b.w to avoid problems.
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
|
||||
.unsupported:
|
||||
mov r0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
.clear:
|
||||
{% for r in range(4, 16) %}
|
||||
veor q{{r}}, q{{r}}, q{{r}}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
+491
@@ -0,0 +1,491 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.arm
|
||||
.text
|
||||
.global armvfpv2_mmm_f32_4x4_{{suffix}}
|
||||
.type armvfpv2_mmm_f32_4x4_{{suffix}}, %function
|
||||
|
||||
// C tile:
|
||||
|
||||
// s16 s20 s24 s28
|
||||
// s17 s21 s25 s29
|
||||
// s18 s22 s26 s30
|
||||
// s19 s23 s27 s31
|
||||
|
||||
// packed A: (2x4) alternating between (s0-s3) and (s4-s7)
|
||||
// packed B: (2x4) alternating between (s8-s11) and (s12-15)
|
||||
|
||||
// all vfp registers in use.
|
||||
|
||||
armvfpv2_mmm_f32_4x4_{{suffix}}:
|
||||
|
||||
/*
|
||||
pld [r1]
|
||||
pld [r1, #8]
|
||||
pld [r2]
|
||||
pld [r2, #8]
|
||||
*/
|
||||
|
||||
push { r4-r12 } // no lr (we're a leaf), no fp. #24 bytes
|
||||
|
||||
ldr r8, [sp, #28]
|
||||
ldr r9, [sp, #24]
|
||||
|
||||
// r8=rsc, r9=csc
|
||||
|
||||
vmrs r6, FPSCR
|
||||
bic r6, r6, #0x00370000
|
||||
vmsr FPSCR, r6
|
||||
|
||||
vpush { s16-s31 }
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.clear:
|
||||
eor r6, r6
|
||||
vmov s16, r6
|
||||
vmov.f32 s17, s16
|
||||
vmov.f32 s18, s16
|
||||
vmov.f32 s19, s16
|
||||
vmov.f32 s20, s16
|
||||
vmov.f32 s21, s16
|
||||
vmov.f32 s22, s16
|
||||
vmov.f32 s23, s16
|
||||
vmov.f32 s24, s16
|
||||
vmov.f32 s25, s16
|
||||
vmov.f32 s26, s16
|
||||
vmov.f32 s27, s16
|
||||
vmov.f32 s28, s16
|
||||
vmov.f32 s29, s16
|
||||
vmov.f32 s30, s16
|
||||
vmov.f32 s31, s16
|
||||
b .non_linear_loop
|
||||
|
||||
.add_mat_mul:
|
||||
// r3 <- k, r4 <- a, r5 <- b
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
mov r1, r4 // packed A ptr
|
||||
pld [r3]
|
||||
pld [r5]
|
||||
|
||||
.packed_packed:
|
||||
cmp r3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.packed_packed_loop_4:
|
||||
|
||||
// 1
|
||||
vldmia r1!, { s0, s1 }
|
||||
vldmia r5!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r1!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r5!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vldmia r1!, { s4-s7 }
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vldmia r5!, { s12-s15 }
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
// 2
|
||||
vmla.f32 s16, s4, s12
|
||||
vmla.f32 s17, s5, s12
|
||||
vmla.f32 s18, s6, s12
|
||||
vmla.f32 s19, s7, s12
|
||||
|
||||
vldmia r1!, { s0-s3 }
|
||||
|
||||
vmla.f32 s20, s4, s13
|
||||
vmla.f32 s21, s5, s13
|
||||
vmla.f32 s22, s6, s13
|
||||
vmla.f32 s23, s7, s13
|
||||
|
||||
vldmia r5!, { s8-s11 }
|
||||
|
||||
vmla.f32 s24, s4, s14
|
||||
vmla.f32 s25, s5, s14
|
||||
vmla.f32 s26, s6, s14
|
||||
vmla.f32 s27, s7, s14
|
||||
|
||||
vmla.f32 s28, s4, s15
|
||||
vmla.f32 s29, s5, s15
|
||||
vmla.f32 s30, s6, s15
|
||||
vmla.f32 s31, s7, s15
|
||||
|
||||
// 3
|
||||
vmla.f32 s16, s0, s8
|
||||
vmla.f32 s17, s1, s8
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vldmia r1!, { s4-s7 }
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vldmia r5!, { s12-s15 }
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
pld [r1]
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
pld [r6]
|
||||
|
||||
// 4
|
||||
vmla.f32 s16, s4, s12
|
||||
vmla.f32 s17, s5, s12
|
||||
vmla.f32 s18, s6, s12
|
||||
vmla.f32 s19, s7, s12
|
||||
|
||||
vmla.f32 s20, s4, s13
|
||||
vmla.f32 s21, s5, s13
|
||||
vmla.f32 s22, s6, s13
|
||||
vmla.f32 s23, s7, s13
|
||||
|
||||
vmla.f32 s24, s4, s14
|
||||
vmla.f32 s25, s5, s14
|
||||
vmla.f32 s26, s6, s14
|
||||
vmla.f32 s27, s7, s14
|
||||
|
||||
vmla.f32 s28, s4, s15
|
||||
vmla.f32 s29, s5, s15
|
||||
vmla.f32 s30, s6, s15
|
||||
vmla.f32 s31, s7, s15
|
||||
|
||||
sub r3, r3, #4
|
||||
cmp r3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp r3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
vldmia r1!, { s0, s1 }
|
||||
vldmia r5!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r1!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r5!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
subs r3, r3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
vldr s0, [ r8 ]
|
||||
vadd.f32 s{{ col * 4 + reg + 16 }}, s{{ col * 4 + reg + 16 }}, s0
|
||||
{% if reg < 3 %}
|
||||
add r8, r8, r4
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_min:
|
||||
vmov s0, r3
|
||||
{% for reg in range(16, 32) %}
|
||||
vcmp.f32 s{{reg}}, s0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge s{{reg}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_max:
|
||||
vmov s0, r3
|
||||
{% for reg in range(16, 32) %}
|
||||
vcmp.f32 s{{reg}}, s0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovle s{{reg}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_add:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vadd.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_mul:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vmul.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vsub.f32 s{{s}}, s0, s{{s}}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub_flipped:
|
||||
vmov s0, r3
|
||||
{% for s in range(16, 32) %}
|
||||
vsub.f32 s{{s}}, s{{s}}, s0
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.leaky_relu:
|
||||
vmov s0, r3
|
||||
{% for reg in range(16, 32) %}
|
||||
vmul.f32 s1, s0, s{{reg}}
|
||||
vcmp.f32 s{{reg}}, #0
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt s{{reg}}, s1
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_min:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{row}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge {{s}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_max:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{row}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt {{s}}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_add:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vadd.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_mul:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vmul.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{row}}, s{{ col * 4 + row + 16 }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub_flipped:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{row}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_min:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{col}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovge {{s}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_max:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
{% set s %}s{{ col * 4 + row + 16 }}{% endset %}
|
||||
vcmp.f32 {{s}}, s{{col}}
|
||||
vmrs apsr_nzcv, fpscr
|
||||
vmovlt {{s}}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_add:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vadd.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_mul:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vmul.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{col}}, s{{ col * 4 + row + 16 }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub_flipped:
|
||||
vldm r3, {s0, s1, s2, s3}
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 4) %}
|
||||
vsub.f32 s{{ col * 4 + row + 16 }}, s{{ col * 4 + row + 16 }}, s{{col}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
vldmia r3!, { s0, s1 }
|
||||
vldmia r4!, { s8, s9 }
|
||||
|
||||
vmla.f32 s16, s0, s8
|
||||
vldmia r3!, { s2, s3 }
|
||||
vmla.f32 s17, s1, s8
|
||||
vldmia r4!, { s10, s11 }
|
||||
vmla.f32 s18, s2, s8
|
||||
vmla.f32 s19, s3, s8
|
||||
|
||||
vmla.f32 s20, s0, s9
|
||||
vmla.f32 s21, s1, s9
|
||||
vmla.f32 s22, s2, s9
|
||||
vmla.f32 s23, s3, s9
|
||||
|
||||
vmla.f32 s24, s0, s10
|
||||
vmla.f32 s25, s1, s10
|
||||
vmla.f32 s26, s2, s10
|
||||
vmla.f32 s27, s3, s10
|
||||
|
||||
vmla.f32 s28, s0, s11
|
||||
vmla.f32 s29, s1, s11
|
||||
vmla.f32 s30, s2, s11
|
||||
vmla.f32 s31, s3, s11
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
{% for col in range(0, 4) %}
|
||||
mov r8, r3
|
||||
{% for reg in range(0, 4) %}
|
||||
fsts s{{ col * 4 + reg + 16 }}, [ r8 ]
|
||||
{% if reg < 3 %}
|
||||
add r8, r8, r4
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if col < 3 %}
|
||||
add r3, r3, r5
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
.load_tile:
|
||||
vldmia r3!, { s16-s31 }
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
|
||||
.return:
|
||||
vpop { s16-s31 }
|
||||
pop { r4-r12 }
|
||||
|
||||
bx lr
|
||||
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
|
||||
.non_linear_loop_entry:
|
||||
sub r0, #20
|
||||
|
||||
.non_linear_loop:
|
||||
add r0, #20
|
||||
ldm r0, { r2, r3, r4, r5, r6 }
|
||||
|
||||
cmp r2, #{{ jump_table | length }}
|
||||
movgt r2, #{{ jump_table | length }}
|
||||
cmp r2, #0
|
||||
movlt r2, #{{ jump_table | length }}
|
||||
|
||||
add pc, pc, r2, LSL#2
|
||||
nop // pc in Rn above is start of the add instruction + 8, hence a nop is needed
|
||||
// This is A32 asm, for T32/Thump2 use nop.w and b.w to avoid problems.
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
.unsupported:
|
||||
mov r0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov r0, #0
|
||||
b .return
|
||||
|
||||
+533
@@ -0,0 +1,533 @@
|
||||
// vim: ft=arm
|
||||
.text
|
||||
.align 4
|
||||
|
||||
/* Z: 32x1
|
||||
z0[0] .. z0[15] z1[0] .. z1[15]
|
||||
*/
|
||||
|
||||
|
||||
.global {{G}}apple_amx_mmm_f16_64x1_{{suffix}}
|
||||
{{G}}apple_amx_mmm_f16_64x1_{{suffix}}:
|
||||
|
||||
{{ AMX_SET }}
|
||||
|
||||
// set x1 to a 128 bytes aligned block for loads
|
||||
mov x1, sp
|
||||
lsr x1, x1, #7
|
||||
lsl x1, x1, #7
|
||||
sub x1, x1, 128
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.leaky_relu:
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x4, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
orr x4, x4, {{ 0 | setting(62) }} // load a pair of A
|
||||
|
||||
mov x5, {{ 0 | setting(43) }} // f16
|
||||
orr x5, x5, {{ 0 | setting(38) }} // Broadcast Y
|
||||
|
||||
orr x6, x5, {{ 0 | setting(20) }} // z offset
|
||||
orr x6, x6, {{ 0 | setting(16) }} // x offset
|
||||
|
||||
cmp x3, #32
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
mov x9, {{ 0 | setting(32) }} // Y broadcast offset += 1
|
||||
|
||||
.packed_packed_loop_32:
|
||||
mov x7, x5
|
||||
mov x8, x6
|
||||
{{ amx("ldy", 2) }}
|
||||
{% for k in range(0, 32) %}
|
||||
{{ amx("ldx", 4) }}
|
||||
add x4, x4, 128
|
||||
{{ amx("vecfp", 7) }}
|
||||
{{ amx("vecfp", 8) }}
|
||||
add x7, x7, x9
|
||||
add x8, x8, x9
|
||||
{% endfor %}
|
||||
add x2, x2, #64
|
||||
sub x3, x3, #32
|
||||
cmp x3, #32
|
||||
bge .packed_packed_loop_32
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
ldr w7, [x2], #2
|
||||
str w7, [x1]
|
||||
{{ amx("ldx", 4) }}
|
||||
{{ amx("ldy", 1) }}
|
||||
{{ amx("vecfp", 5) }}
|
||||
{{ amx("vecfp", 6) }}
|
||||
add x4, x4, 128
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.clear:
|
||||
// top left
|
||||
eor x2, x2, x2
|
||||
orr x2, x2, {{ 0 | setting(27) }}
|
||||
orr x2, x2, {{ 0 | setting(28) }}
|
||||
orr x2, x2, {{ 0 | setting(29) }} // Z = 0
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// top right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 3
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
eor x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub:
|
||||
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
// extr[hxyz] is suport confusing
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }} // extr[hxyz] is confusing
|
||||
{{ amx("fms16", 4) }}
|
||||
|
||||
// continue
|
||||
|
||||
.per_col_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
// broadcast value to x0
|
||||
ld1 { v0.h }[0], [x2]
|
||||
dup v0.8h, v0.h[0]
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldx", 1) }} // load into x0 by default
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(20) }} // target is now z1
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
// broadcast value to x0
|
||||
ld1 { v0.h }[0], [x2]
|
||||
dup v0.8h, v0.h[0]
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldx", 1) }} // load into x0 by default
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
{{ amx("fms16", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(20) }} // target is now z1
|
||||
{{ amx("fms16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2], #64
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1], #64
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // z -= y
|
||||
|
||||
// top left
|
||||
{{ amx("fms16", 2) }}
|
||||
|
||||
// bottom left
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fms16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub:
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
|
||||
// continue
|
||||
|
||||
.per_row_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2], #64
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1], #64
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
// bottom left
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_min:
|
||||
mov x2, 5
|
||||
b .per_row_min_max
|
||||
.per_row_max:
|
||||
mov x2, 7
|
||||
.per_row_min_max:
|
||||
ldr x5, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x5], #64
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1], #64
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x5]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x5, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 5) }}
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(44) }} // f32
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(16) }} // x1
|
||||
orr x2, x2, {{ 0 | setting(20) }} // z1
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_min:
|
||||
mov x2, 5
|
||||
b .per_col_min_max
|
||||
.per_col_max:
|
||||
mov x2, 7
|
||||
.per_col_min_max:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
// broadcast value to x0
|
||||
ld1 { v0.h }[0], [x4]
|
||||
dup v0.8h, v0.h[0]
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(43) }} // f32
|
||||
|
||||
{{ amx("vecfp", 2) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // z offset
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_mul:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
// broadcast value to y0
|
||||
ld1 { v0.h }[0], [x4]
|
||||
dup v0.8h, v0.h[0]
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
st1 { v0.8h }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldy", 1) }}
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
{{ amx("extrx", 2) }}
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
{{ amx("fma16", 4) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z1
|
||||
{{ amx("extrx", 2) }}
|
||||
orr x4, x4, {{ 0 | setting(20) }} // Z1
|
||||
{{ amx("fma16", 4) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_mul:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2], #64
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1], #64
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
{{ amx("extrx", 2) }}
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
{{ amx("fma16", 4) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z1
|
||||
{{ amx("extrx", 2) }}
|
||||
orr x4, x4, {{ 0 | setting(20) }} // Z1
|
||||
orr x4, x4, {{ 0 | setting(6) }} // Y1
|
||||
{{ amx("fma16", 4) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub:
|
||||
// performs a unary neg on Z, then go to scalar_add
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
|
||||
// continue on purpose
|
||||
|
||||
.scalar_add:
|
||||
ldr w5, [x0, #8]
|
||||
|
||||
fmov h0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z+=X
|
||||
{{ amx("fma16", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("fma16", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub_flipped:
|
||||
ldr w5, [x0, #8]
|
||||
fmov s0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z-=X
|
||||
{{ amx("fms16", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("fms16", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_mul:
|
||||
ldr w5, [x0, #8]
|
||||
fmov h0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldy", 1) }}
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma16", 4) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_min:
|
||||
mov x2, 5
|
||||
b .scalar_min_max
|
||||
.scalar_max:
|
||||
mov x2, 7
|
||||
.scalar_min_max:
|
||||
ldr w5, [x0, #8]
|
||||
fmov h0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
lsl x2, x2, 47
|
||||
orr x2, x2, {{ 0 | setting(43) }} // f16
|
||||
|
||||
{{ amx("vecfp", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
{% for neon in range(0, 8) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 { v{{neon}}.h }[{{lane}}], [x5], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
mov x8, x1
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x8], #64
|
||||
st1 { v4.8h, v5.8h, v6.8h, v7.8h }, [x8], #64
|
||||
|
||||
orr x8, x1, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("ldy", 8) }}
|
||||
|
||||
eor x2, x2, x2
|
||||
orr x2, x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // perform Z0+=Y0
|
||||
{{ amx("fma16", 2) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z1
|
||||
orr x2, x2, 64 // offset Y
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x5, x6, [x0, #8] // a base ptr, b base ptr
|
||||
|
||||
ld1 { v0.h }[0], [x6]
|
||||
st1 { v0.h }[0], [x1]
|
||||
{{ amx("ldy", 1) }}
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(43) }} // f16
|
||||
orr x2, x2, {{ 0 | setting(38) }} // Broadcast Y
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(16) }} // X offset
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
ands x8, x5, 0x7f
|
||||
bne .store_generic
|
||||
cmp x6, 4
|
||||
bne .store_generic
|
||||
cmp x7, 4
|
||||
bne .store_generic
|
||||
|
||||
orr x5, x5, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("stz", 5) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
orr x8, x1, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("stz", 8) }}
|
||||
|
||||
mov x8, x1
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x8], #64
|
||||
ld1 { v4.8h, v5.8h, v6.8h, v7.8h }, [x8], #64
|
||||
{% for neon in range(0, 8) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{neon}}.h }[{{lane}}], [x5], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
ldr x2, [x0, #16] // row major ptr
|
||||
orr x2, x2, {{ 0 | setting(62) }} // load pairs
|
||||
{{ amx("ldz", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
{{ AMX_CLR }}
|
||||
ret
|
||||
+658
@@ -0,0 +1,658 @@
|
||||
// vim: ft=arm
|
||||
.text
|
||||
.align 4
|
||||
|
||||
/* Z: 64x32 tile. each Z reg is f16x32
|
||||
Z0
|
||||
Z2
|
||||
...
|
||||
Z62
|
||||
|
||||
Z1
|
||||
Z3
|
||||
S63
|
||||
*/
|
||||
|
||||
|
||||
.global {{G}}apple_amx_mmm_f16_64x32_{{suffix}}
|
||||
{{G}}apple_amx_mmm_f16_64x32_{{suffix}}:
|
||||
|
||||
{{ AMX_SET }}
|
||||
|
||||
// set x1 to a 128 bytes aligned block for loads
|
||||
mov x1, sp
|
||||
lsr x1, x1, #7
|
||||
lsl x1, x1, #7
|
||||
sub x1, x1, 128
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.leaky_relu:
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x4, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
orr x4, x4, {{ 0 | setting(62) }} // load pairs (A)
|
||||
|
||||
eor x5, x5, x5 // top left
|
||||
|
||||
orr x7, x5, {{ 0 | setting(20) }}
|
||||
orr x7, x7, {{ 0 | setting(6) }} // bottom left
|
||||
|
||||
.packed_packed_loop_1:
|
||||
{{ amx("ldx", 2) }}
|
||||
{{ amx("ldy", 4) }}
|
||||
add x2, x2, 64
|
||||
add x4, x4, 128
|
||||
|
||||
{{ amx("fma16", 5) }}
|
||||
{{ amx("fma16", 7) }}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.clear:
|
||||
// top left
|
||||
eor x2, x2, x2
|
||||
orr x2, x2, {{ 0 | setting(27) }}
|
||||
orr x2, x2, {{ 0 | setting(28) }}
|
||||
orr x2, x2, {{ 0 | setting(29) }} // Z = 0
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// top right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 3
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
eor x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
mov x3, #16
|
||||
str x3, [x1]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub:
|
||||
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
mov x6, 64
|
||||
.per_col_sub_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .per_col_sub_loop
|
||||
|
||||
// continue
|
||||
|
||||
.per_col_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
// bottom left
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
|
||||
{{ amx("fms16", 2) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
{{ amx("fms16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2], #64
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1], #64
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(29) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fms16", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 3
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fms16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub:
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
mov x6, 64
|
||||
.per_row_sub_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .per_row_sub_loop
|
||||
|
||||
// continue
|
||||
|
||||
.per_row_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2], #64
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1], #64
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(29) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_min:
|
||||
mov x2, 5
|
||||
b .per_row_min_max
|
||||
.per_row_max:
|
||||
mov x2, 7
|
||||
.per_row_min_max:
|
||||
ldr x5, [x0, #8]
|
||||
|
||||
add x6, x5, 64
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(43) }} // f16
|
||||
|
||||
orr x8, x2, {{ 0 | setting(20) }} // bottom left
|
||||
|
||||
mov x4, 32
|
||||
.loop_per_row_max:
|
||||
// top half
|
||||
ld1 { v0.h }[0], [x5], #2
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
add x2, x2, {{ 0 | setting(21) }}
|
||||
|
||||
// bottom half
|
||||
ld1 { v0.h }[0], [x6], #2
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
{{ amx("vecfp", 8) }}
|
||||
|
||||
add x8, x8, {{ 0 | setting(21) }}
|
||||
|
||||
subs x4, x4, 1
|
||||
bne .loop_per_row_max
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_min:
|
||||
mov x2, 5
|
||||
b .per_col_min_max
|
||||
.per_col_max:
|
||||
mov x2, 7
|
||||
.per_col_min_max:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x4]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(43) }} // f16
|
||||
|
||||
mov x4, 64
|
||||
.loop_per_col_max:
|
||||
{{ amx("vecfp", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }}
|
||||
subs x4, x4, 1
|
||||
bne .loop_per_col_max
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_mul:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x4]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldy", 1) }}
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
|
||||
mov x6, 64
|
||||
.loop_per_col_mul:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }}
|
||||
add x4, x4, {{ 0 | setting(20) }}
|
||||
subs x6, x6, 1
|
||||
bne .loop_per_col_mul
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_mul:
|
||||
ldr x14, [x0, #8]
|
||||
add x15, x14, 64
|
||||
|
||||
// extrx
|
||||
eor x2, x2, x2 // X[0] = Z[0] (top left)
|
||||
|
||||
eor x4, x4, x4
|
||||
orr x4, x4, {{ 0 | setting(20) }} // X[0] = Z[1] (bottom left)
|
||||
|
||||
// fma16
|
||||
eor x6, x6, x6
|
||||
orr x6, x6, {{ 0 | setting(63) }} // vector mode
|
||||
orr x6, x6, {{ 0 | setting(27) }} // Z=X*Y Z[0]=X[0]*Y[0]
|
||||
|
||||
orr x8, x6, {{ 0 | setting(20) }} // Z[1]
|
||||
|
||||
mov x10, 32
|
||||
.loop_per_row_mul:
|
||||
// top
|
||||
ld1 { v0.h }[0], [x14], #2
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
|
||||
{{ amx("ldy", 1) }}
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma16", 6) }}
|
||||
|
||||
add x2, x2, {{ 0 | setting(21) }}
|
||||
add x6, x6, {{ 0 | setting(21) }}
|
||||
|
||||
// bottom
|
||||
ld1 { v0.h }[0], [x15], #2
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
|
||||
{{ amx("ldy", 1) }}
|
||||
{{ amx("extrx", 4) }}
|
||||
{{ amx("fma16", 8) }}
|
||||
|
||||
add x4, x4, {{ 0 | setting(21) }}
|
||||
add x8, x8, {{ 0 | setting(21) }}
|
||||
|
||||
subs x10, x10, 1
|
||||
bne .loop_per_row_mul
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub:
|
||||
// performs a unary neg on Z, then go to scalar_add
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
mov x6, 64
|
||||
.scalar_sub_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .scalar_sub_loop
|
||||
|
||||
// continue on purpose
|
||||
|
||||
.scalar_add:
|
||||
ldr w5, [x0, #8]
|
||||
|
||||
fmov h0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z+=X
|
||||
{{ amx("fma16", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // Z1
|
||||
{{ amx("fma16", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub_flipped:
|
||||
ldr w5, [x0, #8]
|
||||
|
||||
fmov h0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 32 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z-=X
|
||||
{{ amx("fms16", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("fms16", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_mul:
|
||||
ldr w5, [x0, #8]
|
||||
|
||||
fmov h0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldy", 1) }} // load 32 values
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
|
||||
mov x6, 64
|
||||
.scalar_mul_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma16", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .scalar_mul_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_min:
|
||||
mov x2, 5
|
||||
b .scalar_min_max
|
||||
.scalar_max:
|
||||
mov x2, 7
|
||||
.scalar_min_max:
|
||||
ldr w5, [x0, #8]
|
||||
|
||||
fmov h0, w5
|
||||
dup v0.8h, v0.h[0]
|
||||
dup v1.8h, v0.h[0]
|
||||
dup v2.8h, v0.h[0]
|
||||
dup v3.8h, v0.h[0]
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
lsl x2, x2, 47
|
||||
orr x2, x2, {{ 0 | setting(43) }} // f32
|
||||
|
||||
mov x3, 64
|
||||
.loop_scalar_max:
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z
|
||||
{{ amx("vecfp", 2) }}
|
||||
subs x3, x3, 1
|
||||
bne .loop_scalar_max
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
mov x3, 0 // x3 is the row
|
||||
.loop_load:
|
||||
// z reg is (row % 32) * 2 + (row / 32)
|
||||
and x9, x3, 0x1f
|
||||
lsl x9, x9, 1
|
||||
lsr x10, x3, 5
|
||||
add x9, x9, x10
|
||||
|
||||
mov x4, x5
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 { v{{neon}}.h }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
st1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
{{ amx("ldy", 1) }}
|
||||
|
||||
lsl x2, x9, 20 // Z register to update
|
||||
orr x2, x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // perform Z+=Y
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
add x5, x5, x6
|
||||
add x3, x3, 1
|
||||
cmp x3, 64
|
||||
bne .loop_load
|
||||
|
||||
/*
|
||||
mov x3, 0 // x3 is the row
|
||||
.loop_load:
|
||||
and x9, x3, 0xf // x9 = row % 16
|
||||
lsl x9, x9, 2 // x9 = (row % 16) * 4
|
||||
lsr x10, x3, 4 // x10 = row / 16
|
||||
lsl x10, x10, 1 // x10 = (row / 16) * 2
|
||||
add x9, x9, x10 // x9 = x9 + x10
|
||||
|
||||
mov x4, x5
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 { v{{neon}}.s }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 { v{{neon}}.s }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
|
||||
|
||||
mov x2, x1
|
||||
orr x2, x2, {{ 0 | setting(62) }} // load 32 values
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
lsl x2, x9, 20 // left Z register to update
|
||||
orr x2, x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // perform Z+=Y
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
add x2, x2, {{ 0 | setting(20) }}
|
||||
orr x2, x2, 64 // offset Y by 16 values
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
add x5, x5, x6
|
||||
add x3, x3, 1
|
||||
cmp x3, 32
|
||||
bne .loop_load
|
||||
|
||||
*/
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x5, x6, [x0, #8] // a base ptr, b base ptr
|
||||
|
||||
add x8, x1, 64
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x6]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
// top
|
||||
eor x2, x2, x2
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fma16", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x7, 2
|
||||
bne .store_generic
|
||||
ands x8, x5, 0x7f
|
||||
bne .store_generic
|
||||
ands x8, x6, 0x7f
|
||||
bne .store_generic
|
||||
|
||||
lsl x8, x6, 5
|
||||
add x8, x8, x5 // x8 = 32*rsc
|
||||
orr x8, x8, {{ 0 | setting(56) }} // first to x8 is z1
|
||||
|
||||
mov x4, {{ 0 | setting(57) }} // Zreg += 2
|
||||
add x4, x4, x6 // +rsc
|
||||
|
||||
mov x3, 32
|
||||
.loop_store_direct:
|
||||
{{ amx("stz", 5) }}
|
||||
{{ amx("stz", 8) }}
|
||||
add x5, x5, x4
|
||||
add x8, x8, x4
|
||||
subs x3, x3, 1
|
||||
bne .loop_store_direct
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
mov x3, 0 // row id
|
||||
.loop_store:
|
||||
// z reg is (row % 32) * 2 + (row / 32)
|
||||
and x9, x3, 0x1f
|
||||
lsl x9, x9, 1
|
||||
lsr x10, x3, 5
|
||||
add x9, x9, x10
|
||||
|
||||
lsl x2, x9, 56
|
||||
orr x2, x2, x1
|
||||
{{ amx("stz", 2) }} // f16 x 32
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x1]
|
||||
|
||||
mov x4, x5
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{neon}}.h }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x6
|
||||
|
||||
add x3, x3, 1
|
||||
cmp x3, 64
|
||||
bne .loop_store
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
ldr x2, [x0, #16] // row major ptr
|
||||
orr x3, x2, {{ 0 | setting(56) }}
|
||||
add x3, x3, #2048
|
||||
|
||||
mov x4, {{ 0 | setting(57) }} // z+=2
|
||||
add x4, x4, #64
|
||||
|
||||
mov x8, 32
|
||||
.loop_load_tile:
|
||||
{{ amx("ldz", 2) }}
|
||||
{{ amx("ldz", 3) }}
|
||||
add x2, x2, x4
|
||||
add x3, x3, x4
|
||||
subs x8, x8, 1
|
||||
bne .loop_load_tile
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
{{ AMX_CLR }}
|
||||
ret
|
||||
+533
@@ -0,0 +1,533 @@
|
||||
// vim: ft=arm
|
||||
.text
|
||||
.align 4
|
||||
|
||||
/* Z: 32x1
|
||||
z0[0] .. z0[15] z1[0] .. z1[15]
|
||||
*/
|
||||
|
||||
|
||||
.global {{G}}apple_amx_mmm_f32_32x1_{{suffix}}
|
||||
{{G}}apple_amx_mmm_f32_32x1_{{suffix}}:
|
||||
|
||||
{{ AMX_SET }}
|
||||
|
||||
// set x1 to a 128 bytes aligned block for loads
|
||||
mov x1, sp
|
||||
lsr x1, x1, #7
|
||||
lsl x1, x1, #7
|
||||
sub x1, x1, 128
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.leaky_relu:
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x4, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
orr x4, x4, {{ 0 | setting(62) }} // load a pair of A
|
||||
|
||||
mov x5, {{ 0 | setting(44) }} // f32
|
||||
orr x5, x5, {{ 0 | setting(38) }} // Broadcast Y
|
||||
|
||||
orr x6, x5, {{ 0 | setting(20) }} // z offset
|
||||
orr x6, x6, {{ 0 | setting(16) }} // x offset
|
||||
|
||||
cmp x3, #16
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
mov x9, {{ 0 | setting(32) }} // Y broadcast offset += 1
|
||||
|
||||
.packed_packed_loop_16:
|
||||
mov x7, x5
|
||||
mov x8, x6
|
||||
{{ amx("ldy", 2) }}
|
||||
{% for k in range(0, 16) %}
|
||||
{{ amx("ldx", 4) }}
|
||||
add x4, x4, 128
|
||||
{{ amx("vecfp", 7) }}
|
||||
{{ amx("vecfp", 8) }}
|
||||
add x7, x7, x9
|
||||
add x8, x8, x9
|
||||
{% endfor %}
|
||||
add x2, x2, #64
|
||||
sub x3, x3, #16
|
||||
cmp x3, #16
|
||||
bge .packed_packed_loop_16
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1:
|
||||
ldr w7, [x2], #4
|
||||
str w7, [x1]
|
||||
{{ amx("ldx", 4) }}
|
||||
{{ amx("ldy", 1) }}
|
||||
{{ amx("vecfp", 5) }}
|
||||
{{ amx("vecfp", 6) }}
|
||||
add x4, x4, 128
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.clear:
|
||||
// top left
|
||||
eor x2, x2, x2
|
||||
orr x2, x2, {{ 0 | setting(27) }}
|
||||
orr x2, x2, {{ 0 | setting(28) }}
|
||||
orr x2, x2, {{ 0 | setting(29) }} // Z = 0
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// top right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 3
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
eor x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub:
|
||||
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
// extr[hxyz] is suport confusing
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }} // extr[hxyz] is confusing
|
||||
{{ amx("fms32", 4) }}
|
||||
|
||||
// continue
|
||||
|
||||
.per_col_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
// broadcast value to x0
|
||||
ld1 { v0.s }[0], [x2]
|
||||
dup v0.4s, v0.s[0]
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldx", 1) }} // load into x0 by default
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(20) }} // target is now z1
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
// broadcast value to x0
|
||||
ld1 { v0.s }[0], [x2]
|
||||
dup v0.4s, v0.s[0]
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldx", 1) }} // load into x0 by default
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(20) }} // target is now z1
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // z -= y
|
||||
|
||||
// top left
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
// bottom left
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub:
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
|
||||
// continue
|
||||
|
||||
.per_row_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_min:
|
||||
mov x2, 5
|
||||
b .per_row_min_max
|
||||
.per_row_max:
|
||||
mov x2, 7
|
||||
.per_row_min_max:
|
||||
ldr x5, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x5, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 5) }}
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(44) }} // f32
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(16) }} // x1
|
||||
orr x2, x2, {{ 0 | setting(20) }} // z1
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_min:
|
||||
mov x2, 5
|
||||
b .per_col_min_max
|
||||
.per_col_max:
|
||||
mov x2, 7
|
||||
.per_col_min_max:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
// broadcast value to x0
|
||||
ld1 { v0.s }[0], [x4]
|
||||
dup v0.4s, v0.s[0]
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(44) }} // f32
|
||||
|
||||
{{ amx("vecfp", 2) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // z offset
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_mul:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
// broadcast value to y0
|
||||
ld1 { v0.s }[0], [x4]
|
||||
dup v0.4s, v0.s[0]
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
st1 { v0.4s }, [x1], #16
|
||||
sub x1, x1, #64
|
||||
|
||||
{{ amx("ldy", 1) }}
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
{{ amx("extrx", 2) }}
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
{{ amx("fma32", 4) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z1
|
||||
{{ amx("extrx", 2) }}
|
||||
orr x4, x4, {{ 0 | setting(20) }} // Z1
|
||||
{{ amx("fma32", 4) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_mul:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
{{ amx("extrx", 2) }}
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
{{ amx("fma32", 4) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z1
|
||||
{{ amx("extrx", 2) }}
|
||||
orr x4, x4, {{ 0 | setting(20) }} // Z1
|
||||
orr x4, x4, {{ 0 | setting(6) }} // Y1
|
||||
{{ amx("fma32", 4) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub:
|
||||
// performs a unary neg on Z, then go to scalar_add
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
|
||||
// continue on purpose
|
||||
|
||||
.scalar_add:
|
||||
ldr w5, [x0, #8]
|
||||
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z+=X
|
||||
{{ amx("fma32", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("fma32", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub_flipped:
|
||||
ldr w5, [x0, #8]
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z-=X
|
||||
{{ amx("fms32", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("fms32", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_mul:
|
||||
ldr w5, [x0, #8]
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldy", 1) }} // load 16 values
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma32", 4) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_min:
|
||||
mov x2, 5
|
||||
b .scalar_min_max
|
||||
.scalar_max:
|
||||
mov x2, 7
|
||||
.scalar_min_max:
|
||||
ldr w5, [x0, #8]
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
lsl x2, x2, 47
|
||||
orr x2, x2, {{ 0 | setting(44) }} // f32
|
||||
|
||||
{{ amx("vecfp", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
{% for neon in range(0, 8) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 { v{{neon}}.s }[{{lane}}], [x5], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
mov x8, x1
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8], #64
|
||||
st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x8], #64
|
||||
|
||||
orr x8, x1, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("ldy", 8) }}
|
||||
|
||||
eor x2, x2, x2
|
||||
orr x2, x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // perform Z0+=Y0
|
||||
{{ amx("fma32", 2) }}
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z1
|
||||
orr x2, x2, 64 // offset Y by 16 values
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x5, x6, [x0, #8] // a base ptr, b base ptr
|
||||
|
||||
ld1 { v0.s }[0], [x6]
|
||||
st1 { v0.s }[0], [x1]
|
||||
{{ amx("ldy", 1) }}
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(44) }} // f32
|
||||
orr x2, x2, {{ 0 | setting(38) }} // Broadcast Y
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(16) }} // X offset
|
||||
{{ amx("vecfp", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
ands x8, x5, 0x7f
|
||||
bne .store_generic
|
||||
cmp x6, 4
|
||||
bne .store_generic
|
||||
cmp x7, 4
|
||||
bne .store_generic
|
||||
|
||||
orr x5, x5, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("stz", 5) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
orr x8, x1, {{ 0 | setting(62) }} // pair
|
||||
{{ amx("stz", 8) }}
|
||||
|
||||
mov x8, x1
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x8], #64
|
||||
{% for neon in range(0, 8) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{neon}}.s }[{{lane}}], [x5], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
ldr x2, [x0, #16] // row major ptr
|
||||
orr x2, x2, {{ 0 | setting(62) }} // load pairs
|
||||
{{ amx("ldz", 2) }}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
{{ AMX_CLR }}
|
||||
ret
|
||||
+764
@@ -0,0 +1,764 @@
|
||||
// vim: ft=arm
|
||||
.text
|
||||
.align 4
|
||||
|
||||
/* Z: 32x32
|
||||
z0[0] .. z0[15] z1[0] .. z1[15]
|
||||
z4[0] .. z4[15] z5[0] .. z5[15]
|
||||
..
|
||||
z60[0] .. z60[15] z61[0] .. z61[15]
|
||||
|
||||
z2[0] .. z2[15] z3[0] .. z3[15]
|
||||
z5[0] .. z5[15] z6[0] .. z6[15]
|
||||
..
|
||||
z62[0] .. z62[15] z63[0] .. z63[15]
|
||||
*/
|
||||
|
||||
|
||||
.global {{G}}apple_amx_mmm_f32_32x32_{{suffix}}
|
||||
{{G}}apple_amx_mmm_f32_32x32_{{suffix}}:
|
||||
|
||||
{{ AMX_SET }}
|
||||
|
||||
// set x1 to a 128 bytes aligned block for loads
|
||||
mov x1, sp
|
||||
lsr x1, x1, #7
|
||||
lsl x1, x1, #7
|
||||
sub x1, x1, 128
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.leaky_relu:
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
|
||||
.add_mat_mul:
|
||||
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x4, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
orr x4, x4, {{ 0 | setting(62) }} // load pairs (A)
|
||||
orr x2, x2, {{ 0 | setting(62) }} // load pairs (B)
|
||||
|
||||
eor x5, x5, x5 // top left
|
||||
|
||||
orr x6, x5, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x6, x6, {{ 0 | setting(16) }} // top right
|
||||
|
||||
orr x7, x5, {{ 0 | setting(21) }}
|
||||
orr x7, x7, {{ 0 | setting(6) }} // bottom left
|
||||
|
||||
orr x8, x7, x6 // bottom right
|
||||
|
||||
.packed_packed_loop_1:
|
||||
{{ amx("ldx", 2) }}
|
||||
{{ amx("ldy", 4) }}
|
||||
add x2, x2, 128
|
||||
add x4, x4, 128
|
||||
|
||||
{{ amx("fma32", 5) }}
|
||||
{{ amx("fma32", 6) }}
|
||||
{{ amx("fma32", 7) }}
|
||||
{{ amx("fma32", 8) }}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.clear:
|
||||
// top left
|
||||
eor x2, x2, x2
|
||||
orr x2, x2, {{ 0 | setting(27) }}
|
||||
orr x2, x2, {{ 0 | setting(28) }}
|
||||
orr x2, x2, {{ 0 | setting(29) }} // Z = 0
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// top right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 3
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
eor x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub:
|
||||
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
mov x6, 64
|
||||
.per_col_sub_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .per_col_sub_loop
|
||||
|
||||
// continue
|
||||
|
||||
.per_col_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x1, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 2
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(16) }} // X offset
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 3
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// top right
|
||||
eor x2, x2, {{ 0 | setting(21) }} // Z row = 1
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x1, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 1) }}
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
// bottom left
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 2
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(16) }} // X offset
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 3
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
// top right
|
||||
eor x2, x2, {{ 0 | setting(21) }} // Z row = 1
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub_flipped:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(29) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
// top right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 3
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
// bottom left
|
||||
eor x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
{{ amx("fms32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_sub:
|
||||
// performs a unary neg on Z
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
mov x6, 64
|
||||
.per_row_sub_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .per_row_sub_loop
|
||||
|
||||
// continue
|
||||
|
||||
.per_row_add:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x2]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
mov x2, {{ 0 | setting(29) }} // z += y
|
||||
|
||||
// top left
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// top right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 3
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
eor x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_min:
|
||||
mov x2, 5
|
||||
b .per_row_min_max
|
||||
.per_row_max:
|
||||
mov x2, 7
|
||||
.per_row_min_max:
|
||||
ldr x5, [x0, #8]
|
||||
|
||||
add x6, x5, 64
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(44) }} // f32
|
||||
orr x3, x2, {{ 0 | setting(20) }} // right half: z offset
|
||||
|
||||
orr x8, x2, {{ 0 | setting(21) }} // bottom left
|
||||
orr x9, x3, {{ 0 | setting(21) }} // bottom right
|
||||
|
||||
mov x4, 16
|
||||
.loop_per_row_max:
|
||||
// top half
|
||||
ld1 { v0.s }[0], [x5], #4
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
{{ amx("vecfp", 2) }}
|
||||
{{ amx("vecfp", 3) }}
|
||||
|
||||
add x2, x2, {{ 0 | setting(22) }}
|
||||
add x3, x3, {{ 0 | setting(22) }}
|
||||
|
||||
// bottom half
|
||||
ld1 { v0.s }[0], [x6], #4
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
|
||||
{{ amx("ldx", 1) }}
|
||||
{{ amx("vecfp", 8) }}
|
||||
{{ amx("vecfp", 9) }}
|
||||
|
||||
add x8, x8, {{ 0 | setting(22) }}
|
||||
add x9, x9, {{ 0 | setting(22) }}
|
||||
|
||||
subs x4, x4, 1
|
||||
bne .loop_per_row_max
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_min:
|
||||
mov x2, 5
|
||||
b .per_col_min_max
|
||||
.per_col_max:
|
||||
mov x2, 7
|
||||
.per_col_min_max:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x4], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x4]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x3, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 3) }}
|
||||
|
||||
lsl x2, x2, 47 // max(x,z) (or min)
|
||||
orr x2, x2, {{ 0 | setting(44) }} // f32
|
||||
|
||||
orr x3, x2, {{ 0 | setting(16) }} // right half: x offset
|
||||
orr x3, x3, {{ 0 | setting(20) }} // right half: z offset
|
||||
|
||||
mov x4, 32
|
||||
.loop_per_col_max:
|
||||
{{ amx("vecfp", 2) }}
|
||||
{{ amx("vecfp", 3) }}
|
||||
add x2, x2, {{ 0 | setting(21) }}
|
||||
add x3, x3, {{ 0 | setting(21) }}
|
||||
subs x4, x4, 1
|
||||
bne .loop_per_col_max
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_col_mul:
|
||||
ldr x4, [x0, #8]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x4], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x4]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
eor x3, x3, x3
|
||||
orr x3, x3, {{ 0 | setting(20) }} // Z[1]
|
||||
orr x3, x3, {{ 0 | setting(16) }} // X[1]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
|
||||
mov x5, {{ 0 | setting(63) }} // vector mode
|
||||
orr x5, x5, {{ 0 | setting(27) }} // Z=X*Y
|
||||
orr x5, x5, {{ 0 | setting(20) }} // Z right
|
||||
orr x5, x5, {{ 0 | setting(16) }} // X[1] (right)
|
||||
orr x5, x5, {{ 0 | setting(6) }} // Y[1] (right)
|
||||
|
||||
mov x6, 32
|
||||
.loop_per_col_mul:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("extrx", 3) }}
|
||||
{{ amx("fma32", 4) }}
|
||||
{{ amx("fma32", 5) }}
|
||||
add x2, x2, {{ 0 | setting(21) }}
|
||||
add x3, x3, {{ 0 | setting(21) }}
|
||||
add x4, x4, {{ 0 | setting(21) }}
|
||||
add x5, x5, {{ 0 | setting(21) }}
|
||||
subs x6, x6, 1
|
||||
bne .loop_per_col_mul
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.per_row_mul:
|
||||
ldr x14, [x0, #8]
|
||||
add x15, x14, 64
|
||||
|
||||
// extrx
|
||||
eor x2, x2, x2 // X[0] = Z[0] (top left)
|
||||
|
||||
eor x3, x3, x3
|
||||
orr x3, x3, {{ 0 | setting(20) }} // Z[1]
|
||||
orr x3, x3, {{ 0 | setting(16) }} // X[1] = Z[1] (top right)
|
||||
|
||||
eor x4, x4, x4
|
||||
orr x4, x4, {{ 0 | setting(21) }} // X[0] = Z[2] (bottom left)
|
||||
|
||||
orr x5, x4, {{ 0 | setting(20) }}
|
||||
orr x5, x5, {{ 0 | setting(16) }} // X[1] = Z[3] (bottom right)
|
||||
|
||||
// fma32
|
||||
eor x6, x6, x6
|
||||
orr x6, x6, {{ 0 | setting(63) }} // vector mode
|
||||
orr x6, x6, {{ 0 | setting(27) }} // Z=X*Y Z[0]=X[0]*Y[0]
|
||||
|
||||
orr x7, x6, {{ 0 | setting(20) }} // Z[1]
|
||||
orr x7, x7, {{ 0 | setting(16) }} // X[1] Z[1] = X[1]*Y[0]
|
||||
|
||||
orr x8, x6, {{ 0 | setting(21) }} // Z[2]
|
||||
orr x8, x8, {{ 0 | setting(21) }} // Z[2]
|
||||
|
||||
orr x9, x8, {{ 0 | setting(20) }} // Z[3]
|
||||
orr x9, x9, {{ 0 | setting(16) }} // X[1]
|
||||
|
||||
mov x10, 16
|
||||
.loop_per_row_mul:
|
||||
// top
|
||||
ld1 { v0.s }[0], [x14], #4
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
|
||||
{{ amx("ldy", 1) }}
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("extrx", 3) }}
|
||||
{{ amx("fma32", 6) }}
|
||||
{{ amx("fma32", 7) }}
|
||||
|
||||
add x2, x2, {{ 0 | setting(22) }}
|
||||
add x3, x3, {{ 0 | setting(22) }}
|
||||
add x6, x6, {{ 0 | setting(22) }}
|
||||
add x7, x7, {{ 0 | setting(22) }}
|
||||
|
||||
// bottom
|
||||
ld1 { v0.s }[0], [x15], #4
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
|
||||
{{ amx("ldy", 1) }}
|
||||
{{ amx("extrx", 4) }}
|
||||
{{ amx("extrx", 5) }}
|
||||
{{ amx("fma32", 8) }}
|
||||
{{ amx("fma32", 9) }}
|
||||
|
||||
add x4, x4, {{ 0 | setting(22) }}
|
||||
add x5, x5, {{ 0 | setting(22) }}
|
||||
add x8, x8, {{ 0 | setting(22) }}
|
||||
add x9, x9, {{ 0 | setting(22) }}
|
||||
|
||||
subs x10, x10, 1
|
||||
bne .loop_per_row_mul
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub:
|
||||
// performs a unary neg on Z, then go to scalar_add
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(28) }}
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=-X
|
||||
|
||||
mov x6, 64
|
||||
.scalar_sub_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fms32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .scalar_sub_loop
|
||||
|
||||
// continue on purpose
|
||||
|
||||
.scalar_add:
|
||||
ldr w5, [x0, #8]
|
||||
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z+=X
|
||||
{% for chunk in range(0, 4) %}
|
||||
{{ amx("fma32", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_sub_flipped:
|
||||
ldr w5, [x0, #8]
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
mov x2, {{ 0 | setting(28) }} // Z-=X
|
||||
{% for chunk in range(0, 4) %}
|
||||
{{ amx("fms32", 2) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_mul:
|
||||
ldr w5, [x0, #8]
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldy", 1) }} // load 16 values
|
||||
|
||||
eor x2, x2, x2 // X[0] = Z[0]
|
||||
|
||||
mov x4, {{ 0 | setting(63) }} // vector mode
|
||||
orr x4, x4, {{ 0 | setting(27) }} // Z=X*Y
|
||||
|
||||
mov x6, 64
|
||||
.scalar_mul_loop:
|
||||
{{ amx("extrx", 2) }}
|
||||
{{ amx("fma32", 4) }}
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z row
|
||||
add x4, x4, {{ 0 | setting(20) }} // next Z row
|
||||
subs x6, x6, 1
|
||||
bne .scalar_mul_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.scalar_min:
|
||||
mov x2, 5
|
||||
b .scalar_min_max
|
||||
.scalar_max:
|
||||
mov x2, 7
|
||||
.scalar_min_max:
|
||||
ldr w5, [x0, #8]
|
||||
fmov s0, w5
|
||||
dup v0.4s, v0.s[0]
|
||||
dup v1.4s, v0.s[0]
|
||||
dup v2.4s, v0.s[0]
|
||||
dup v3.4s, v0.s[0]
|
||||
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{{ amx("ldx", 1) }} // load 16 values
|
||||
|
||||
lsl x2, x2, 47
|
||||
orr x2, x2, {{ 0 | setting(44) }} // f32
|
||||
|
||||
mov x3, 64
|
||||
.loop_scalar_max:
|
||||
add x2, x2, {{ 0 | setting(20) }} // next Z
|
||||
{{ amx("vecfp", 2) }}
|
||||
subs x3, x3, 1
|
||||
bne .loop_scalar_max
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
add x8, x1, 64
|
||||
|
||||
mov x3, 0 // x3 is the row
|
||||
.loop_load:
|
||||
and x9, x3, 0xf // x9 = row % 16
|
||||
lsl x9, x9, 2 // x9 = (row % 16) * 4
|
||||
lsr x10, x3, 4 // x10 = row / 16
|
||||
lsl x10, x10, 1 // x10 = (row / 16) * 2
|
||||
add x9, x9, x10 // x9 = x9 + x10
|
||||
|
||||
mov x4, x5
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 { v{{neon}}.s }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 { v{{neon}}.s }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
|
||||
|
||||
mov x2, x1
|
||||
orr x2, x2, {{ 0 | setting(62) }} // load 32 values
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
lsl x2, x9, 20 // left Z register to update
|
||||
orr x2, x2, {{ 0 | setting(63) }} // vector mode
|
||||
orr x2, x2, {{ 0 | setting(29) }} // perform Z+=Y
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
add x2, x2, {{ 0 | setting(20) }}
|
||||
orr x2, x2, 64 // offset Y by 16 values
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
add x5, x5, x6
|
||||
add x3, x3, 1
|
||||
cmp x3, 32
|
||||
bne .loop_load
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x5, x6, [x0, #8] // a base ptr, b base ptr
|
||||
|
||||
add x8, x1, 64
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x5]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldy", 2) }}
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x6], #64
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1], #64
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x6]
|
||||
st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
sub x1, x1, #64
|
||||
|
||||
orr x2, x1, {{ 0 | setting(62) }} // load a pair
|
||||
{{ amx("ldx", 2) }}
|
||||
|
||||
// top left
|
||||
eor x2, x2, x2
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// top right
|
||||
orr x2, x2, {{ 0 | setting(20) }} // Z row = 1
|
||||
orr x2, x2, {{ 0 | setting(16) }} // X offset
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom right
|
||||
orr x2, x2, {{ 0 | setting(21) }} // Z row = 3
|
||||
orr x2, x2, {{ 0 | setting(6) }} // Y offset
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
// bottom left
|
||||
eor x2, x2, {{ 0 | setting(20) }} // Z row = 2
|
||||
eor x2, x2, {{ 0 | setting(16) }} // X offset <-
|
||||
{{ amx("fma32", 2) }}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x7, 4
|
||||
bne .store_generic
|
||||
ands x8, x5, 0x7f
|
||||
bne .store_generic
|
||||
ands x8, x6, 0x7f
|
||||
bne .store_generic
|
||||
|
||||
orr x5, x5, {{ 0 | setting(62) }} // pair
|
||||
lsl x8, x6, 4
|
||||
add x8, x8, x5 // x8 = 16*rsc
|
||||
orr x8, x8, {{ 0 | setting(57) }} // first to x8 is z2
|
||||
|
||||
mov x4, {{ 0 | setting(58) }} // Zreg += 4
|
||||
add x4, x4, x6 // +rsc
|
||||
|
||||
mov x3, 16
|
||||
.loop_store_direct:
|
||||
{{ amx("stz", 5) }}
|
||||
{{ amx("stz", 8) }}
|
||||
add x5, x5, x4
|
||||
add x8, x8, x4
|
||||
subs x3, x3, 1
|
||||
bne .loop_store_direct
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_generic:
|
||||
|
||||
add x8, x1, 64
|
||||
|
||||
mov x3, 0 // row id
|
||||
.loop_store:
|
||||
and x9, x3, 0xf // x9 = row % 16
|
||||
lsl x9, x9, 2 // x9 = (row % 16) * 4
|
||||
lsr x10, x3, 4 // x10 = row / 16
|
||||
lsl x10, x10, 1 // x10 = (row / 16) * 2
|
||||
add x9, x9, x10 // x9 = x9 + x10
|
||||
|
||||
lsl x2, x9, 56
|
||||
orr x2, x2, {{ 0 | setting(62) }}
|
||||
orr x2, x2, x1
|
||||
{{ amx("stz", 2) }}
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x1]
|
||||
|
||||
mov x4, x5
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{neon}}.s }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
|
||||
{% for neon in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{neon}}.s }[{{lane}}], [x4], x7
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x6
|
||||
|
||||
add x3, x3, 1
|
||||
cmp x3, 32
|
||||
bne .loop_store
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.load_tile:
|
||||
ldr x2, [x0, #16] // row major ptr
|
||||
orr x2, x2, {{ 0 | setting(62) }} // load pairs
|
||||
mov x3, x2
|
||||
orr x3, x3, {{ 0 | setting(57) }}
|
||||
add x3, x3, #2048
|
||||
|
||||
mov x4, {{ 0 | setting(58) }} // z+=4
|
||||
add x4, x4, #128
|
||||
|
||||
mov x8, 16
|
||||
.loop_load_tile:
|
||||
{{ amx("ldz", 2) }}
|
||||
{{ amx("ldz", 3) }}
|
||||
add x2, x2, x4
|
||||
add x3, x3, x4
|
||||
subs x8, x8, 1
|
||||
bne .loop_load_tile
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
{{ AMX_CLR }}
|
||||
ret
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
sub x0, x0, 40
|
||||
|
||||
.non_linear_loop:
|
||||
add x0, x0, 40
|
||||
ldr x2, [x0]
|
||||
|
||||
mov x4, #{{ jump_table | length }}
|
||||
|
||||
cmp x2, #{{ jump_table | length }}
|
||||
csel x2, x2, x4, lt
|
||||
cmp x2, #0
|
||||
csel x2, x4, x2, lt
|
||||
|
||||
adr x3, .jmp_table
|
||||
add x3, x3, x2, LSL#2
|
||||
br x3
|
||||
|
||||
.jmp_table:
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
add x0, x2, #4000
|
||||
b .return
|
||||
|
||||
.unsupported:
|
||||
mov x0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov x0, 0
|
||||
b .return
|
||||
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_leaky_relu_f16_8n_{{suffix}}
|
||||
{{G}}arm64fp16_leaky_relu_f16_8n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
mov v31.h[0], w2
|
||||
dup v31.8h, v31.h[0]
|
||||
mov x2, x0
|
||||
|
||||
cmp x1, #64
|
||||
blt .loop
|
||||
|
||||
ld1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x2], #64
|
||||
.loop4:
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [x2], #64
|
||||
|
||||
fmul v20.8h, v16.8h, v31.8h
|
||||
fmul v21.8h, v17.8h, v31.8h
|
||||
fmul v22.8h, v18.8h, v31.8h
|
||||
fmul v23.8h, v19.8h, v31.8h
|
||||
|
||||
fcmge v24.8h, v16.8h, #0.0
|
||||
fcmge v25.8h, v17.8h, #0.0
|
||||
fcmge v26.8h, v18.8h, #0.0
|
||||
fcmge v27.8h, v19.8h, #0.0
|
||||
|
||||
bsl v24.16b, v16.16b, v20.16b
|
||||
bsl v25.16b, v17.16b, v21.16b
|
||||
bsl v26.16b, v18.16b, v22.16b
|
||||
bsl v27.16b, v19.16b, v23.16b
|
||||
|
||||
st1 { v24.8h, v25.8h, v26.8h, v27.8h }, [x0], #64
|
||||
|
||||
and v16.16b, v0.16b, v0.16b
|
||||
and v17.16b, v1.16b, v1.16b
|
||||
and v18.16b, v2.16b, v2.16b
|
||||
and v19.16b, v3.16b, v3.16b
|
||||
|
||||
subs x1, x1, #32
|
||||
cmp x1, #64
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.8h }, [x0]
|
||||
|
||||
fmul v17.8h, v16.8h, v31.8h
|
||||
fcmge v18.8h, v16.8h, #0.0
|
||||
bsl v18.16b, v16.16b, v17.16b
|
||||
|
||||
st1 { v18.8h }, [x0], #16
|
||||
|
||||
subs x1, x1, #8
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
|
||||
{% macro scalar(label, op, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
add x2, x0, #8
|
||||
ld1 {v0.h}[0], [ x2 ]
|
||||
dup v0.8h, v0.h[0]
|
||||
{% if flipped %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} v{{reg}}.8h, v{{reg}}.8h, v0.8h
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} v{{reg}}.8h, v0.8h, v{{reg}}.8h
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_row(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% set mr_over_8 = mr // 8 %}
|
||||
{% set mr_over_8_min_1 = mr // 8 - 1 %}
|
||||
|
||||
{% for reg in range(0, mr_over_8_min_1 + 1) %}
|
||||
ldr q{{reg}}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_8 %}
|
||||
{{op}} v{{acc}}.8h, v{{acc}}.8h, v{{other}}.8h
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_8 %}
|
||||
{{op}} v{{acc}}.8h, v{{other}}.8h, v{{acc}}.8h
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_col(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% set mr_over_8 = mr // 8 %}
|
||||
{% set cols = (to + 1 - from) // mr_over_8 %}
|
||||
|
||||
{% set loads = cols // 8 %}
|
||||
|
||||
{%if cols == 1 %}
|
||||
ld1 {v0.h}[0], [ x2 ]
|
||||
{% elif cols == 3 %}
|
||||
ld1 {v0.s}[0], [ x2 ], #4
|
||||
ld1 {v0.h}[2], [ x2 ]
|
||||
{% elif cols == 4 %}
|
||||
ldr d0, [ x2 ]
|
||||
{% elif cols == 6 %}
|
||||
ldr d0, [ x2 ], #8
|
||||
ld1 {v0.s}[2], [ x2 ]
|
||||
{% else %}
|
||||
{% for reg in range(1, loads + 1) %}
|
||||
ldr q{{ reg - 1 }}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
// mr:{{mr}} {{ loads }} {{cols}}
|
||||
|
||||
{% for col in range(1, cols + 1) %}
|
||||
dup v3.8h, v{{ (col - 1) // 8 }}.h[{{ (col - 1) % 8 }}]
|
||||
{% for row in range(1, mr_over_8 + 1) %}
|
||||
{% set acc = (col - 1) * mr_over_8 + row - 1 + from %}
|
||||
{% if flipped %}
|
||||
{{op}} v{{acc}}.8h, v{{acc}}.8h, v3.8h
|
||||
{% else %}
|
||||
{{op}} v{{acc}}.8h, v3.8h, v{{acc}}.8h
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x5, [x1, #128]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr x6, [x1, #136]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x7, [x1, #144]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr x9, [x1, #152]
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x10, [x1, #96]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr x11, [x1, #104]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x12, [x1, #112]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr x13, [x1, #120]
|
||||
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [ x1 ]
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
ldr x14, [x1, #128]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
ldr x15, [x1, #136]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
ldr x20, [x1, #144]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
ldr x21, [x1, #152]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
ldr x22, [x1, #160]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
ldr x23, [x1, #168]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
ldr x24, [x1, #176]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
ldr x25, [x1, #184]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v1.d[0], x7
|
||||
ins v2.d[0], x10
|
||||
ins v3.d[0], x12
|
||||
ins v4.d[0], x14
|
||||
ins v5.d[0], x20
|
||||
ins v6.d[0], x22
|
||||
ins v7.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v1.d[1], x9
|
||||
ins v2.d[1], x11
|
||||
ins v3.d[1], x13
|
||||
ins v4.d[1], x15
|
||||
ins v5.d[1], x21
|
||||
ins v6.d[1], x23
|
||||
ins v7.d[1], x25
|
||||
|
||||
add x1, x1, #192
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
ld1 {{ v9.8h, v10.8h, v11.8h, v12.8h }}, [x1], #64
|
||||
ld1 {{ v13.8h, v14.8h, v15.8h }}, [x1], #48
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
fmla v24.8h, v9.8h, v8.h[0]
|
||||
ld1 {{ v9.8h }}, [ x1 ], #16
|
||||
ld1 {{ v0.8h, v1.8h, v2.8h, v3.8h }}, [x1], #64
|
||||
ld1 {{ v4.8h, v5.8h, v6.8h, v7.8h }}, [x1], #64
|
||||
fmla v25.8h, v10.8h, v8.h[0]
|
||||
fmla v26.8h, v11.8h, v8.h[0]
|
||||
fmla v27.8h, v12.8h, v8.h[0]
|
||||
fmla v28.8h, v13.8h, v8.h[0]
|
||||
fmla v29.8h, v14.8h, v8.h[0]
|
||||
fmla v30.8h, v15.8h, v8.h[0]
|
||||
|
||||
fmla v31.8h, v9.8h, v8.h[0]
|
||||
|
||||
ld1 {{ v8.h }}[0], [ x2 ], #2
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s }}, [x1], #48
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ldr w8, [x2], #4
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
ldr d12, [x1], #8
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
ldr x12, [x1], #8
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
ldr d14, [x1], #8
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
ldr x14, [x1], #8
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
ldr d15, [x1], #8
|
||||
fmla v24.8h, v9.8h, v8.h[0]
|
||||
ldr x15, [x1], #8
|
||||
|
||||
ld1 {{ v0.8h, v1.8h, v2.8h, v3.8h }}, [x1], #64
|
||||
ins v8.s[1], w8
|
||||
ld1 {{ v4.8h, v5.8h, v6.8h, v7.8h }}, [x1], #64
|
||||
|
||||
fmla v25.8h, v10.8h, v8.h[0]
|
||||
ins v12.d[1], x12
|
||||
fmla v26.8h, v11.8h, v8.h[0]
|
||||
ins v13.d[1], x13
|
||||
fmla v27.8h, v12.8h, v8.h[0]
|
||||
ins v14.d[1], x14
|
||||
fmla v28.8h, v13.8h, v8.h[0]
|
||||
ins v15.d[1], x15
|
||||
|
||||
ld1 {{ v9.8h, v10.8h, v11.8h, v12.8h }}, [x1], #64
|
||||
|
||||
fmla v29.8h, v14.8h, v8.h[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v30.8h, v15.8h, v8.h[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v31.8h, v0.8h, v8.h[0]
|
||||
ldr d14, [x1], #8
|
||||
|
||||
fmla v16.8h, v1.8h, v8.h[2]
|
||||
ldr x14, [x1], #8
|
||||
fmla v17.8h, v2.8h, v8.h[2]
|
||||
ldr d15, [x1], #8
|
||||
fmla v18.8h, v3.8h, v8.h[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v19.8h, v4.8h, v8.h[2]
|
||||
|
||||
ld1 {{ v0.8h }}, [x1], #16
|
||||
|
||||
fmla v20.8h, v5.8h, v8.h[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.8h, v6.8h, v8.h[2]
|
||||
ldr x10, [x1], #8
|
||||
|
||||
fmla v22.8h, v7.8h, v8.h[2]
|
||||
|
||||
fmla v23.8h, v9.8h, v8.h[2]
|
||||
ins v13.d[1], x13
|
||||
fmla v24.8h, v10.8h, v8.h[2]
|
||||
ins v14.d[1], x14
|
||||
fmla v25.8h, v11.8h, v8.h[2]
|
||||
ins v15.d[1], x15
|
||||
|
||||
fmla v26.8h, v12.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
fmla v27.8h, v13.8h, v8.h[2]
|
||||
ins v1.d[1], x10
|
||||
fmla v28.8h, v14.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
fmla v29.8h, v15.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
fmla v30.8h, v0.8h, v8.h[2]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
fmla v31.8h, v1.8h, v8.h[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.h[0], v8.h[3]
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// v8..v15 are callee-preserved
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_128x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_128x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
sub x3, x3, #1
|
||||
|
||||
|
||||
ld1 { v8.h }[0], [ x2 ], #2
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% include "arm64fp16_mmm_f16_128x1/loop1/naive.S.raw" %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% include "arm64fp16_mmm_f16_128x1/loop2/cortex_a55.S.raw" %}
|
||||
{% endset %}
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
// last loop can't read beyond actual input as it's likely not packed and padded
|
||||
.packed_packed_loop_1_last:
|
||||
ld1 { v9.8h, v10.8h, v11.8h, v12.8h }, [x1], #64
|
||||
ld1 { v13.8h, v14.8h, v15.8h }, [x1], #48
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
ld1 { v0.8h }, [ x1 ]
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
|
||||
fmla v24.8h, v9.8h, v8.h[0]
|
||||
fmla v25.8h, v10.8h, v8.h[0]
|
||||
fmla v26.8h, v11.8h, v8.h[0]
|
||||
fmla v27.8h, v12.8h, v8.h[0]
|
||||
fmla v28.8h, v13.8h, v8.h[0]
|
||||
fmla v29.8h, v14.8h, v8.h[0]
|
||||
fmla v30.8h, v15.8h, v8.h[0]
|
||||
fmla v31.8h, v0.8h, v8.h[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_scalars.j2" %}
|
||||
{% set mr = 128 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_rows.j2" %}
|
||||
{% set mr = 128 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #2
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.8h, v{{reg}}.8h, v0.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.8h-v3.8h}, [x5], #64
|
||||
ld1 {v4.8h-v7.8h}, [x5], #64
|
||||
ld1 {v8.8h-v11.8h}, [x5], #64
|
||||
ld1 {v12.8h-v15.8h}, [x5], #64
|
||||
|
||||
{% for r in range(0, 16) %}
|
||||
fadd v{{ r + 16 }}.8h, v{{ r + 16 }}.8h, v{{r}}.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.h}[0], [ x3 ]
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
ldr q{{r}}, [x2], #16
|
||||
{% endfor %}
|
||||
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ldr q0, [x2], #16
|
||||
fmla v17.8h, v1.8h, v8.h[0]
|
||||
ldr q1, [x2], #16
|
||||
fmla v18.8h, v2.8h, v8.h[0]
|
||||
ldr q2, [x2], #16
|
||||
fmla v19.8h, v3.8h, v8.h[0]
|
||||
ldr q3, [x2], #16
|
||||
fmla v20.8h, v4.8h, v8.h[0]
|
||||
ldr q4, [x2], #16
|
||||
fmla v21.8h, v5.8h, v8.h[0]
|
||||
ldr q5, [x2], #16
|
||||
fmla v22.8h, v6.8h, v8.h[0]
|
||||
ldr q6, [x2], #16
|
||||
fmla v23.8h, v7.8h, v8.h[0]
|
||||
ldr q7, [x2], #16
|
||||
|
||||
fmla v24.8h, v0.8h, v8.h[0]
|
||||
fmla v25.8h, v1.8h, v8.h[0]
|
||||
fmla v26.8h, v2.8h, v8.h[0]
|
||||
fmla v27.8h, v3.8h, v8.h[0]
|
||||
fmla v28.8h, v4.8h, v8.h[0]
|
||||
fmla v29.8h, v5.8h, v8.h[0]
|
||||
fmla v30.8h, v6.8h, v8.h[0]
|
||||
fmla v31.8h, v7.8h, v8.h[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc$
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{reg}}.h }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
st1 { v{{reg}}.8h }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v0.8h, v4.h[1]
|
||||
fmla v19.8h, v1.8h, v4.h[1]
|
||||
fmla v20.8h, v0.8h, v4.h[2]
|
||||
fmla v21.8h, v1.8h, v4.h[2]
|
||||
fmla v22.8h, v0.8h, v4.h[3]
|
||||
fmla v23.8h, v1.8h, v4.h[3]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[4]
|
||||
fmla v25.8h, v1.8h, v4.h[4]
|
||||
fmla v26.8h, v0.8h, v4.h[5]
|
||||
fmla v27.8h, v1.8h, v4.h[5]
|
||||
fmla v28.8h, v0.8h, v4.h[6]
|
||||
fmla v29.8h, v1.8h, v4.h[6]
|
||||
fmla v30.8h, v0.8h, v4.h[7]
|
||||
fmla v31.8h, v1.8h, v4.h[7]
|
||||
|
||||
ld1 {{ v0.8h, v1.8h }}, [x1], #32
|
||||
ld1 {{ v4.8h }}, [x2], #16
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
ldr d2, [x1], #8
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
ldr d6, [x2], #8
|
||||
fmla v18.8h, v0.8h, v4.h[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v1.8h, v4.h[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.8h, v0.8h, v4.h[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v21.8h, v1.8h, v4.h[2]
|
||||
fmla v22.8h, v0.8h, v4.h[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.8h, v1.8h, v4.h[3]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[4]
|
||||
fmla v25.8h, v1.8h, v4.h[4]
|
||||
fmla v26.8h, v0.8h, v4.h[5]
|
||||
fmla v27.8h, v1.8h, v4.h[5]
|
||||
fmla v28.8h, v0.8h, v4.h[6]
|
||||
ins v2.d[1], x5
|
||||
fmla v29.8h, v1.8h, v4.h[6]
|
||||
ins v6.d[1], x7
|
||||
fmla v30.8h, v0.8h, v4.h[7]
|
||||
ins v3.d[1], x6
|
||||
fmla v31.8h, v1.8h, v4.h[7]
|
||||
|
||||
fmla v16.8h, v2.8h, v6.h[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.8h, v3.8h, v6.h[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.8h, v2.8h, v6.h[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v3.8h, v6.h[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.8h, v2.8h, v6.h[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.8h, v3.8h, v6.h[2]
|
||||
fmla v22.8h, v2.8h, v6.h[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.8h, v3.8h, v6.h[3]
|
||||
|
||||
fmla v24.8h, v2.8h, v6.h[4]
|
||||
fmla v25.8h, v3.8h, v6.h[4]
|
||||
fmla v26.8h, v2.8h, v6.h[5]
|
||||
fmla v27.8h, v3.8h, v6.h[5]
|
||||
fmla v28.8h, v2.8h, v6.h[6]
|
||||
ins v0.d[1], x5
|
||||
fmla v29.8h, v3.8h, v6.h[6]
|
||||
ins v4.d[1], x7
|
||||
fmla v30.8h, v2.8h, v6.h[7]
|
||||
ins v1.d[1], x6
|
||||
fmla v31.8h, v3.8h, v6.h[7]
|
||||
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// x20..x27 are used, callee-preserved
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_16x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_16x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
str q8, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed:
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s }, [ x2 ], #16
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% include "arm64fp16_mmm_f16_16x8/loop1/naive.S.raw" %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64fp16_mmm_f16_16x8/loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_scalars.j2" %}
|
||||
{% set mr = 16 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_rows.j2" %}
|
||||
{% set mr = 16 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 2 + reg }}.8h, v{{ col * 2 + reg }}.8h, v0.8h
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ], #32
|
||||
ld1 { v4.4s }, [ x3 ], #16
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v0.8h, v4.h[1]
|
||||
fmla v19.8h, v1.8h, v4.h[1]
|
||||
fmla v20.8h, v0.8h, v4.h[2]
|
||||
fmla v21.8h, v1.8h, v4.h[2]
|
||||
fmla v22.8h, v0.8h, v4.h[3]
|
||||
fmla v23.8h, v1.8h, v4.h[3]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[4]
|
||||
fmla v25.8h, v1.8h, v4.h[4]
|
||||
fmla v26.8h, v0.8h, v4.h[5]
|
||||
fmla v27.8h, v1.8h, v4.h[5]
|
||||
fmla v28.8h, v0.8h, v4.h[6]
|
||||
fmla v29.8h, v1.8h, v4.h[6]
|
||||
fmla v30.8h, v0.8h, v4.h[7]
|
||||
fmla v31.8h, v1.8h, v4.h[7]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #2
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
str q{{ col * 2 }}, [ x5 ]
|
||||
str q{{ col * 2 + 1 }}, [ x5, #16 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{ col * 2 + reg }}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldr q8, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v2.8h, v4.h[0]
|
||||
fmla v19.8h, v3.8h, v4.h[0]
|
||||
fmla v20.8h, v0.8h, v4.h[1]
|
||||
fmla v21.8h, v1.8h, v4.h[1]
|
||||
fmla v22.8h, v2.8h, v4.h[1]
|
||||
fmla v23.8h, v3.8h, v4.h[1]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[2]
|
||||
fmla v25.8h, v1.8h, v4.h[2]
|
||||
fmla v26.8h, v2.8h, v4.h[2]
|
||||
fmla v27.8h, v3.8h, v4.h[2]
|
||||
fmla v28.8h, v0.8h, v4.h[3]
|
||||
fmla v29.8h, v1.8h, v4.h[3]
|
||||
fmla v30.8h, v2.8h, v4.h[3]
|
||||
fmla v31.8h, v3.8h, v4.h[3]
|
||||
|
||||
ld1 {{ v0.8h, v1.8h, v2.8h, v3.8h }}, [ x1 ], #64
|
||||
ldr d4, [x2], #8
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
// mul a: v0, v1, v2, v3 b: v4
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9 as d9
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.8h, v2.8h, v4.h[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v3.8h, v4.h[0]
|
||||
fmla v20.8h, v0.8h, v4.h[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.8h, v1.8h, v4.h[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.8h, v2.8h, v4.h[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.8h, v3.8h, v4.h[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.8h, v1.8h, v4.h[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.8h, v2.8h, v4.h[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.8h, v3.8h, v4.h[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.8h, v0.8h, v4.h[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.8h, v1.8h, v4.h[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.8h, v2.8h, v4.h[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.8h, v3.8h, v4.h[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4 as d4
|
||||
|
||||
fmla v16.8h, v5.8h, v9.h[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.8h, v6.8h, v9.h[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.8h, v7.8h, v9.h[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.8h, v8.8h, v9.h[0]
|
||||
fmla v20.8h, v5.8h, v9.h[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.8h, v6.8h, v9.h[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.8h, v7.8h, v9.h[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.8h, v8.8h, v9.h[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.8h, v5.8h, v9.h[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.8h, v6.8h, v9.h[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.8h, v7.8h, v9.h[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.8h, v8.8h, v9.h[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.8h, v5.8h, v9.h[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.8h, v6.8h, v9.h[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.8h, v7.8h, v9.h[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.8h, v8.8h, v9.h[3]
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// x20..x27 are used, callee-preserved
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_32x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_32x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ldr d4, [x2], #8
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% include "arm64fp16_mmm_f16_32x4/loop1/naive.S.raw" %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64fp16_mmm_f16_32x4/loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64fp16_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 4 + 16 + reg }}.8h, v{{ col * 4 + 16 + reg }}.8h, v0.8h
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x2 ]
|
||||
ldr d4, [x3]
|
||||
|
||||
fmla v16.8h, v0.8h, v4.h[0]
|
||||
fmla v17.8h, v1.8h, v4.h[0]
|
||||
fmla v18.8h, v2.8h, v4.h[0]
|
||||
fmla v19.8h, v3.8h, v4.h[0]
|
||||
fmla v20.8h, v0.8h, v4.h[1]
|
||||
fmla v21.8h, v1.8h, v4.h[1]
|
||||
fmla v22.8h, v2.8h, v4.h[1]
|
||||
fmla v23.8h, v3.8h, v4.h[1]
|
||||
|
||||
fmla v24.8h, v0.8h, v4.h[2]
|
||||
fmla v25.8h, v1.8h, v4.h[2]
|
||||
fmla v26.8h, v2.8h, v4.h[2]
|
||||
fmla v27.8h, v3.8h, v4.h[2]
|
||||
fmla v28.8h, v0.8h, v4.h[3]
|
||||
fmla v29.8h, v1.8h, v4.h[3]
|
||||
fmla v30.8h, v2.8h, v4.h[3]
|
||||
fmla v31.8h, v3.8h, v4.h[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #2
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
str q{{ col * 4 + 16 + 0 }}, [ x5 ]
|
||||
str q{{ col * 4 + 16 + 1 }}, [ x5, #16 ]
|
||||
str q{{ col * 4 + 16 + 2 }}, [ x5, #32 ]
|
||||
str q{{ col * 4 + 16 + 3 }}, [ x5, #48 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{ col * 4 + 16 + reg }}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// v8..v15 are callee-preserved
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_32x6_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_32x6_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
ld1 { v7.8h }, [ x2 ]
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x1 ], #64
|
||||
add x2, x2, 12
|
||||
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 6) %}
|
||||
fmla v{{ col * 4 + 8 + row }}.8h, v{{row}}.8h, v7.h[{{col}}]
|
||||
{% endfor %}
|
||||
/*
|
||||
{% for col in range(0, 2) %}
|
||||
fmla v{{ (col + 4) * 4 + 8 + row }}.8h, v{{row}}.8h, v6.h[{{col}}]
|
||||
{% endfor %}
|
||||
*/
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 6) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 4 + 8 + reg }}.8h, v{{ col * 4 + 8 + reg }}.8h, v0.8h
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.8h-v3.8h}, [x5], #64
|
||||
ld1 {v4.8h-v7.8h}, [x5], #64
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
fadd v{{ r + 24 }}.8h, v{{ r + 24 }}.8h, v{{r}}.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x2, x3, [x0, #8]
|
||||
|
||||
ld1 { v7.d }[0], [ x3 ], #8
|
||||
ld1 { v7.s }[2], [ x3 ], #4
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x2 ], #64
|
||||
|
||||
{% for row in range(0, 4) %}
|
||||
{% for col in range(0, 6) %}
|
||||
fmla v{{ col * 4 + 8 + row }}.8h, v{{row}}.8h, v7.h[{{col}}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for col in range(0, 6) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{ col * 4 + 8 + reg }}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for col in range(0, 6) %}
|
||||
mov x4, x5
|
||||
{% for r in range(0, 4) %}
|
||||
st1 { v{{ col * 4 + 8 + r }}.8h }, [ x4 ], 16
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+264
@@ -0,0 +1,264 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// v8..v15 are callee-preserved
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_64x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_64x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .q4f16se
|
||||
|
||||
cmp x4, #2
|
||||
beq .q4f16
|
||||
|
||||
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
ld1 { v8.h }[0], [ x2 ], #2
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x1 ], #64
|
||||
ld1 { v4.8h, v5.8h, v6.8h, v7.8h }, [ x1 ], #64
|
||||
|
||||
fmla v24.8h, v0.8h, v8.h[0]
|
||||
fmla v25.8h, v1.8h, v8.h[0]
|
||||
fmla v26.8h, v2.8h, v8.h[0]
|
||||
fmla v27.8h, v3.8h, v8.h[0]
|
||||
fmla v28.8h, v4.8h, v8.h[0]
|
||||
fmla v29.8h, v5.8h, v8.h[0]
|
||||
fmla v30.8h, v6.8h, v8.h[0]
|
||||
fmla v31.8h, v7.8h, v8.h[0]
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 8
|
||||
.q40f16_const:
|
||||
.byte 0xc8, 0xc7, 0xc6, 0xc5, 0xc4, 0xc2, 0xc0, 0xbc
|
||||
.byte 0x00, 0x3c, 0x40, 0x42, 0x44, 0x45, 0x46, 0x47
|
||||
|
||||
.q4f16se:
|
||||
adr x4, .q40f16_const
|
||||
movi v15.16b, 15
|
||||
ld1 {v13.16b}, [ x4 ]
|
||||
eor v12.16b, v12.16b, v12.16b
|
||||
|
||||
.q4f16se_outerloop:
|
||||
{% for i in range(0, 8) %}
|
||||
eor v{{ i + 16 }}.16b, v{{ i + 16 }}.16b, v{{ i + 16 }}.16b
|
||||
{% endfor %}
|
||||
mov x4, #32
|
||||
|
||||
.p2align 4
|
||||
.q4f16se_innerloop:
|
||||
ld1 { v9.16b-v10.16b }, [x1], #32
|
||||
ld1 { v8.h }[0], [ x2 ], #2
|
||||
|
||||
and v0.16b, v9.16b, v15.16b
|
||||
ushr v2.16b, v9.16b, 4
|
||||
|
||||
and v4.16b, v10.16b, v15.16b
|
||||
ushr v6.16b, v10.16b, 4
|
||||
|
||||
tbl v0.16b, { v13.16b }, v0.16b
|
||||
tbl v2.16b, { v13.16b }, v2.16b
|
||||
tbl v4.16b, { v13.16b }, v4.16b
|
||||
tbl v6.16b, { v13.16b }, v6.16b
|
||||
|
||||
zip2 v1.16b, v12.16b, v0.16b
|
||||
zip2 v3.16b, v12.16b, v2.16b
|
||||
zip2 v5.16b, v12.16b, v4.16b
|
||||
zip2 v7.16b, v12.16b, v6.16b
|
||||
|
||||
zip1 v0.16b, v12.16b, v0.16b
|
||||
zip1 v2.16b, v12.16b, v2.16b
|
||||
zip1 v4.16b, v12.16b, v4.16b
|
||||
zip1 v6.16b, v12.16b, v6.16b
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 16 }}.8h, v{{i}}.8h, v8.h[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x4, x4, #1
|
||||
bne .q4f16se_innerloop
|
||||
|
||||
// scales
|
||||
ld1 { v0.8h-v3.8h }, [ x1 ], #64
|
||||
ld1 { v4.8h-v7.8h }, [ x1 ], #64
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 24 }}.8h, v{{i}}.8h, v{{ i + 16 }}.8h
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #32
|
||||
bne .q4f16se_outerloop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q4f16:
|
||||
adr x4, .q40f16_const
|
||||
movi v15.16b, 15
|
||||
ld1 {v13.16b}, [ x4 ]
|
||||
eor v12.16b, v12.16b, v12.16b
|
||||
|
||||
.q4f16_outerloop:
|
||||
// scales
|
||||
ld1 { v16.8h-v19.8h }, [ x1 ], #64
|
||||
ld1 { v20.8h-v23.8h }, [ x1 ], #64
|
||||
mov x4, #32
|
||||
|
||||
.p2align 4
|
||||
.q4f16_innerloop:
|
||||
ld1 { v9.16b-v10.16b }, [x1], #32
|
||||
ld1 { v8.h }[0], [ x2 ], #2
|
||||
|
||||
and v0.16b, v9.16b, v15.16b
|
||||
ushr v2.16b, v9.16b, 4
|
||||
|
||||
and v4.16b, v10.16b, v15.16b
|
||||
ushr v6.16b, v10.16b, 4
|
||||
|
||||
tbl v0.16b, { v13.16b }, v0.16b
|
||||
tbl v2.16b, { v13.16b }, v2.16b
|
||||
tbl v4.16b, { v13.16b }, v4.16b
|
||||
tbl v6.16b, { v13.16b }, v6.16b
|
||||
|
||||
zip2 v1.16b, v12.16b, v0.16b
|
||||
zip2 v3.16b, v12.16b, v2.16b
|
||||
zip2 v5.16b, v12.16b, v4.16b
|
||||
zip2 v7.16b, v12.16b, v6.16b
|
||||
|
||||
zip1 v0.16b, v12.16b, v0.16b
|
||||
zip1 v2.16b, v12.16b, v2.16b
|
||||
zip1 v4.16b, v12.16b, v4.16b
|
||||
zip1 v6.16b, v12.16b, v6.16b
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmul v{{i}}.8h, v{{i}}.8h, v{{ i + 16 }}.8h
|
||||
{% endfor %}
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 24 }}.8h, v{{i}}.8h, v8.h[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x4, x4, #1
|
||||
bne .q4f16_innerloop
|
||||
|
||||
subs x3, x3, #32
|
||||
bne .q4f16_outerloop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 24 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_scalars.j2" %}
|
||||
{% set mr = 64 %}{% set from = 24 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_rows.j2" %}
|
||||
{% set mr = 64 %}{% set from = 24 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_cols.j2" %}
|
||||
{% set from = 24 %}{% set to = 31 %}{% include "arm64fp16_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #2
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.8h, v{{reg}}.8h, v0.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.8h-v3.8h}, [x5], #64
|
||||
ld1 {v4.8h-v7.8h}, [x5], #64
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
fadd v{{ r + 24 }}.8h, v{{ r + 24 }}.8h, v{{r}}.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.h}[0], [ x3 ]
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
ldr q{{r}}, [x2], #16
|
||||
{% endfor %}
|
||||
|
||||
fmla v24.8h, v0.8h, v8.h[0]
|
||||
fmla v25.8h, v1.8h, v8.h[0]
|
||||
fmla v26.8h, v2.8h, v8.h[0]
|
||||
fmla v27.8h, v3.8h, v8.h[0]
|
||||
fmla v28.8h, v4.8h, v8.h[0]
|
||||
fmla v29.8h, v5.8h, v8.h[0]
|
||||
fmla v30.8h, v6.8h, v8.h[0]
|
||||
fmla v31.8h, v7.8h, v8.h[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc$
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{reg}}.h }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
st1 { v{{reg}}.8h }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// v8..v15 are callee-preserved
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_mmm_f16_64x3_{{core}}_{{suffix}}
|
||||
{{G}}arm64fp16_mmm_f16_64x3_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
ld1 { v7.4s }, [ x2 ]
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x1 ], #64
|
||||
ld1 { v4.8h, v5.8h, v6.8h }, [ x1 ], #48
|
||||
add x2, x2, #6
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.8h, v0.8h, v7.h[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.8h }, [ x1 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.8h, v{{row}}.8h, v7.h[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.8h, v0.8h, v7.h[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_scalars.j2" %}
|
||||
{% set mr = 64 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_rows.j2" %}
|
||||
{% set mr = 64 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_f16_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64fp16_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 8) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
ld1 {v0.h}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 8 + 8 + reg }}.8h, v{{ col * 8 + 8 + reg }}.8h, v0.8h
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.8h-v3.8h}, [x5], #64
|
||||
ld1 {v4.8h-v7.8h}, [x5], #64
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
fadd v{{ r + 24 }}.8h, v{{ r + 24 }}.8h, v{{r}}.8h
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x2, x3, [x0, #8]
|
||||
|
||||
ld1 { v7.s }[0], [ x3 ], #4
|
||||
ld1 { v7.h }[2], [ x3 ], #2
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x2 ], #64
|
||||
ld1 { v4.8h, v5.8h, v6.8h }, [ x2 ], #48
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.8h, v0.8h, v7.h[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.8h }, [ x2 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.8h, v{{row}}.8h, v7.h[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.8h, v0.8h, v7.h[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 8) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{ col * 8 + 8 + reg }}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for r in range(0, 8) %}
|
||||
st1 { v{{ col * 8 + 8 + r }}.8h }, [ x4 ], 16
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64fp16_mmm_8h_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col(label="per_col_min", op="fmin", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_max", op="fmax", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_mul", op="fmul", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_add", op="fadd", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub", op="fsub", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub_flipped", op="fsub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64fp16_mmm_8h_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row(label="per_row_min", op="fmin", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_max", op="fmax", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_mul", op="fmul", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_add", op="fadd", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub", op="fsub", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub_flipped", op="fsub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64fp16_mmm_8h_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar(label="scalar_min", op="fmin", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_max", op="fmax", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_mul", op="fmul", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_add", op="fadd", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub", op="fsub", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub_flipped", op="fsub", from=from, to=to, flipped=true) }}
|
||||
|
||||
.clear:
|
||||
{% for r in range(from, to + 1) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.leaky_relu:
|
||||
add x2, x0, #8
|
||||
ld1 {v4.s}[0], [ x2 ]
|
||||
dup v4.8h, v4.h[0]
|
||||
|
||||
// bsl cond/dst, then, else
|
||||
// fcmge dst, src, #0.0
|
||||
{% for r in range(from, to + 1) %}
|
||||
fmul v0.8h, v{{r}}.8h, v4.8h
|
||||
fcmge v1.8h, v{{r}}.8h, #0.0
|
||||
bsl v1.16b, v{{r}}.16b, v0.16b
|
||||
and v{{r}}.16b, v1.16b, v1.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.load_tile:
|
||||
ldr x2, [ x0, #8 ]
|
||||
{% for reg in range(from, to + 1) %}
|
||||
ld1 { v{{reg}}.4s }, [ x2 ], #16
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_sigmoid_f16_8n_{{suffix}}
|
||||
{{G}}arm64fp16_sigmoid_f16_8n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.8h }, [x2]
|
||||
dup v5.8h, v0.h[0] // v5 <- low, broadcasted
|
||||
dup v6.8h, v0.h[1] // v6 <- high, broadcasted
|
||||
dup v7.8h, v0.h[7] // v7 <- half, broadcasted
|
||||
|
||||
cmp x1, #32
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmax v17.8h, v17.8h, v5.8h
|
||||
fmax v18.8h, v18.8h, v5.8h
|
||||
fmax v19.8h, v19.8h, v5.8h
|
||||
|
||||
fmin v16.8h, v16.8h, v6.8h
|
||||
fmin v17.8h, v17.8h, v6.8h
|
||||
fmin v18.8h, v18.8h, v6.8h
|
||||
fmin v19.8h, v19.8h, v6.8h // v16 <- x
|
||||
|
||||
fmul v20.8h, v16.8h, v16.8h
|
||||
fmul v21.8h, v17.8h, v17.8h
|
||||
fmul v22.8h, v18.8h, v18.8h
|
||||
fmul v23.8h, v19.8h, v19.8h // v20 <- x2
|
||||
|
||||
dup v28.8h, v0.h[3]
|
||||
fmla v28.8h, v20.8h, v0.h[2]
|
||||
dup v29.8h, v0.h[3]
|
||||
fmla v29.8h, v21.8h, v0.h[2]
|
||||
dup v30.8h, v0.h[3]
|
||||
fmla v30.8h, v22.8h, v0.h[2]
|
||||
dup v31.8h, v0.h[3]
|
||||
fmla v31.8h, v23.8h, v0.h[2]
|
||||
|
||||
dup v24.8h, v0.h[4]
|
||||
fmla v24.8h, v20.8h, v28.8h
|
||||
dup v25.8h, v0.h[4]
|
||||
fmla v25.8h, v21.8h, v29.8h
|
||||
dup v26.8h, v0.h[4]
|
||||
fmla v26.8h, v22.8h, v30.8h
|
||||
dup v27.8h, v0.h[4]
|
||||
fmla v27.8h, v23.8h, v31.8h
|
||||
|
||||
fmul v16.8h, v16.8h, v24.8h
|
||||
fmul v17.8h, v17.8h, v25.8h
|
||||
fmul v18.8h, v18.8h, v26.8h
|
||||
fmul v19.8h, v19.8h, v27.8h // v16 <- numerator
|
||||
|
||||
dup v24.8h, v0.h[6]
|
||||
dup v25.8h, v0.h[6]
|
||||
dup v26.8h, v0.h[6]
|
||||
dup v27.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v0.h[5]
|
||||
fmla v25.8h, v21.8h, v0.h[5]
|
||||
fmla v26.8h, v22.8h, v0.h[5]
|
||||
fmla v27.8h, v23.8h, v0.h[5] // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
fdiv v17.8h, v17.8h, v25.8h
|
||||
fdiv v18.8h, v18.8h, v26.8h
|
||||
fdiv v19.8h, v19.8h, v27.8h
|
||||
|
||||
fadd v16.8h, v16.8h, v7.8h
|
||||
fadd v17.8h, v17.8h, v7.8h
|
||||
fadd v18.8h, v18.8h, v7.8h
|
||||
fadd v19.8h, v19.8h, v7.8h
|
||||
|
||||
st1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0], #64
|
||||
|
||||
subs x1, x1, #32
|
||||
cmp x1, #32
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmin v16.8h, v16.8h, v6.8h // v16 <- x
|
||||
fmul v20.8h, v16.8h, v16.8h // v20 <- x2
|
||||
|
||||
dup v28.8h, v0.h[3]
|
||||
fmla v28.8h, v20.8h, v0.h[2]
|
||||
dup v24.8h, v0.h[4]
|
||||
fmla v24.8h, v20.8h, v28.8h
|
||||
fmul v16.8h, v16.8h, v24.8h // v16 <- numerator
|
||||
|
||||
dup v24.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v0.h[5] // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
fadd v16.8h, v16.8h, v7.8h
|
||||
|
||||
st1 { v16.8h }, [x0], #16
|
||||
|
||||
subs x1, x1, #8
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
{{ -6.92 | float16 }}
|
||||
{{ 6.92 | float16 }}
|
||||
{{ -0.0000124702 | float16 }}
|
||||
{{ 0.00400222 | float16 }}
|
||||
|
||||
{{ 0.249895 | float16 }}
|
||||
{{ 0.098734 | float16 }}
|
||||
{{ 1.0 | float16 }}
|
||||
{{ 0.5 | float16 }}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// no preservation either for v0-v7 and v16-v31
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
{% if needs_pragma == true %}
|
||||
.cpu generic+fp+simd+fp16
|
||||
{% endif %}
|
||||
.global {{G}}arm64fp16_tanh_f16_8n_{{suffix}}
|
||||
{{G}}arm64fp16_tanh_f16_8n_{{suffix}}:
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
adr x2, .coeffs_num
|
||||
ld1 { v0.8h }, [x2]
|
||||
dup v5.8h, v0.h[0] // v5 <- low, broadcasted
|
||||
dup v6.8h, v0.h[1] // v6 <- high, broadcasted
|
||||
|
||||
cmp x1, #32
|
||||
blt .loop
|
||||
|
||||
.loop4:
|
||||
ld1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmax v17.8h, v17.8h, v5.8h
|
||||
fmax v18.8h, v18.8h, v5.8h
|
||||
fmax v19.8h, v19.8h, v5.8h
|
||||
|
||||
fmin v16.8h, v16.8h, v6.8h
|
||||
fmin v17.8h, v17.8h, v6.8h
|
||||
fmin v18.8h, v18.8h, v6.8h
|
||||
fmin v19.8h, v19.8h, v6.8h // v16 <- x
|
||||
|
||||
fmul v20.8h, v16.8h, v16.8h
|
||||
fmul v21.8h, v17.8h, v17.8h
|
||||
fmul v22.8h, v18.8h, v18.8h
|
||||
fmul v23.8h, v19.8h, v19.8h // v20 <- x2
|
||||
|
||||
dup v24.8h, v0.h[3]
|
||||
fmla v24.8h, v20.8h, v0.h[2]
|
||||
dup v25.8h, v0.h[3]
|
||||
fmla v25.8h, v21.8h, v0.h[2]
|
||||
dup v26.8h, v0.h[3]
|
||||
fmla v26.8h, v22.8h, v0.h[2]
|
||||
dup v27.8h, v0.h[3]
|
||||
fmla v27.8h, v23.8h, v0.h[2]
|
||||
|
||||
fmul v16.8h, v16.8h, v24.8h
|
||||
fmul v17.8h, v17.8h, v25.8h
|
||||
fmul v18.8h, v18.8h, v26.8h
|
||||
fmul v19.8h, v19.8h, v27.8h // v16 <- numerator
|
||||
|
||||
dup v28.8h, v0.h[5]
|
||||
fmla v28.8h, v20.8h, v0.h[4]
|
||||
dup v29.8h, v0.h[5]
|
||||
fmla v29.8h, v21.8h, v0.h[4]
|
||||
dup v30.8h, v0.h[5]
|
||||
fmla v30.8h, v22.8h, v0.h[4]
|
||||
dup v31.8h, v0.h[5]
|
||||
fmla v31.8h, v23.8h, v0.h[4]
|
||||
|
||||
dup v24.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v28.8h
|
||||
dup v25.8h, v0.h[6]
|
||||
fmla v25.8h, v21.8h, v29.8h
|
||||
dup v26.8h, v0.h[6]
|
||||
fmla v26.8h, v22.8h, v30.8h
|
||||
dup v27.8h, v0.h[6]
|
||||
fmla v27.8h, v23.8h, v31.8h // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
fdiv v17.8h, v17.8h, v25.8h
|
||||
fdiv v18.8h, v18.8h, v26.8h
|
||||
fdiv v19.8h, v19.8h, v27.8h
|
||||
|
||||
st1 { v16.8h, v17.8h, v18.8h, v19.8h }, [x0], #64
|
||||
|
||||
subs x1, x1, #32
|
||||
cmp x1, #32
|
||||
bge .loop4
|
||||
|
||||
cmp x1, #0
|
||||
beq .return
|
||||
|
||||
.loop:
|
||||
ld1 { v16.8h }, [x0]
|
||||
|
||||
fmax v16.8h, v16.8h, v5.8h
|
||||
fmin v16.8h, v16.8h, v6.8h // v16 <- x
|
||||
fmul v20.8h, v16.8h, v16.8h // v20 <- x2
|
||||
|
||||
dup v24.8h, v0.h[3]
|
||||
fmla v24.8h, v20.8h, v0.h[2]
|
||||
fmul v16.8h, v16.8h, v24.8h // v16 <- numerator
|
||||
|
||||
dup v28.8h, v0.h[5]
|
||||
fmla v28.8h, v20.8h, v0.h[4]
|
||||
dup v24.8h, v0.h[6]
|
||||
fmla v24.8h, v20.8h, v28.8h // v24 <- denum
|
||||
|
||||
fdiv v16.8h, v16.8h, v24.8h
|
||||
|
||||
st1 { v16.8h }, [x0], #16
|
||||
|
||||
subs x1, x1, #8
|
||||
bne .loop
|
||||
|
||||
.return:
|
||||
ret
|
||||
|
||||
.coeffs_num:
|
||||
{{ -3.84 | float16 }}
|
||||
{{ 3.84 | float16 }}
|
||||
{{ 0.082654955 | float16 }} // alpha
|
||||
{{ 0.99963124 | float16 }}
|
||||
|
||||
{{ 0.0065383179 | float16 }} // beta
|
||||
{{ 0.41401828 | float16 }}
|
||||
{{ 1.0 | float16 }}
|
||||
{{ 0 | float16 }} // padding
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
// vim: ft=arm
|
||||
|
||||
.non_linear:
|
||||
sub x0, x0, 40
|
||||
|
||||
.non_linear_loop:
|
||||
add x0, x0, 40
|
||||
ldr x2, [x0]
|
||||
|
||||
mov x4, #{{ jump_table | length }}
|
||||
|
||||
cmp x2, #{{ jump_table | length }}
|
||||
csel x2, x2, x4, lt
|
||||
cmp x2, #0
|
||||
csel x2, x4, x2, lt
|
||||
|
||||
adr x3, .jmp_table
|
||||
add x3, x3, x2, LSL#2
|
||||
br x3
|
||||
|
||||
.jmp_table:
|
||||
{% for j in jump_table %}
|
||||
b .{{j}}
|
||||
{% endfor %}
|
||||
b .unsupported
|
||||
|
||||
add x0, x2, #4000
|
||||
b .return
|
||||
|
||||
.unsupported:
|
||||
mov x0, #1
|
||||
b .return
|
||||
|
||||
.done:
|
||||
mov x0, 0
|
||||
b .return
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// serves as a canary build file to figure out which flag combination will accept half precision fmla
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
// .cpu generic+fp+simd+fp16
|
||||
.global foo
|
||||
foo:
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ret
|
||||
|
||||
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// serves as a canary build file to figure out which flag combination will accept half precision fmla
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd+fp16
|
||||
.global foo
|
||||
foo:
|
||||
fmla v16.8h, v0.8h, v8.h[0]
|
||||
ret
|
||||
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
|
||||
{% macro scalar(label, op, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
add x2, x0, #8
|
||||
ld1 {v0.s}[0], [ x2 ]
|
||||
dup v0.4s, v0.s[0]
|
||||
{% if flipped %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for reg in range(from, to + 1) %}
|
||||
{{op}} v{{reg}}.4s, v0.4s, v{{reg}}.4s
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_row(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set mr_over_4_min_1 = mr // 4 - 1 %}
|
||||
|
||||
{% for reg in range(0, mr_over_4_min_1 + 1) %}
|
||||
ldr q{{reg}}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
|
||||
{% if flipped %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} v{{acc}}.4s, v{{acc}}.4s, v{{other}}.4s
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
{% for acc in range(from, to + 1) %}
|
||||
{% set other = (acc - from) % mr_over_4 %}
|
||||
{{op}} v{{acc}}.4s, v{{other}}.4s, v{{acc}}.4s
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
|
||||
{% macro per_col(label, op, mr, from, to, flipped=false) %}
|
||||
.{{label}}:
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
{% set mr_over_4 = mr // 4 %}
|
||||
{% set cols = (to + 1 - from) // mr_over_4 %}
|
||||
|
||||
{% set loads = cols // 4 %}
|
||||
|
||||
{%if cols == 1 %}
|
||||
ld1 {v0.s}[0], [ x2 ]
|
||||
{% elif cols == 3 %}
|
||||
ldr d0, [ x2 ], #8
|
||||
ld1 {v0.s}[2], [ x2 ]
|
||||
{% else %}
|
||||
{% for reg in range(1, loads + 1) %}
|
||||
ldr q{{ reg - 1 }}, [ x2 ], #16
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
// {{mr}} {{cols}}
|
||||
|
||||
{% for col in range(1, cols + 1) %}
|
||||
dup v3.4s, v{{ (col - 1) // 4 }}.s[{{ (col - 1) % 4 }}]
|
||||
{% for row in range(1, mr_over_4 + 1) %}
|
||||
{% set acc = (col - 1) * mr_over_4 + row - 1 + from %}
|
||||
{% if flipped %}
|
||||
{{op}} v{{acc}}.4s, v{{acc}}.4s, v3.4s
|
||||
{% else %}
|
||||
{{op}} v{{acc}}.4s, v3.4s, v{{acc}}.4s
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
{% endmacro %}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr w4, [x1], #4
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr w22, [x2], #4
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr w23, [x2], #4
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr w8, [x1], #4
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr w24, [x2], #4
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr w9, [x1], #4
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
ldr w25, [x2], #4
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
ldr w10, [x1], #4
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ldr w26, [x2], #4
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
ldr w11, [x1], #4
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
ldr w27, [x2], #4
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
ldr w12, [x1], #4
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
ldr w13, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
ldr w14, [x1], #4
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
ldr w15, [x1], #4
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ins v0.s[0], w4
|
||||
ins v1.s[0], w8
|
||||
ins v2.s[0], w12
|
||||
ins v4.s[0], w20
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[1], w5
|
||||
ins v1.s[1], w9
|
||||
ins v2.s[1], w13
|
||||
ins v4.s[1], w21
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[2], w6
|
||||
ins v1.s[2], w10
|
||||
ins v2.s[2], w14
|
||||
ins v4.s[2], w22
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[3], w7
|
||||
ins v1.s[3], w11
|
||||
ins v2.s[3], w15
|
||||
ins v4.s[3], w23
|
||||
ins v5.s[3], w27
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr w4, [x1]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr w5, [x1, #4]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr w6, [x1, #8]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr w22, [x2], #4
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1, #12]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr w23, [x2], #4
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr w8, [x1, #16]
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr w24, [x2], #4
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr w9, [x1, #20]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
ldr w25, [x2], #4
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
ldr w10, [x1, #24]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ldr w26, [x2], #4
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
ldr w11, [x1, #28]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
ldr w27, [x2], #4
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
ldr w12, [x1, #32]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
ldr w13, [x1, #36]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
ldr w14, [x1, #40]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
ldr w15, [x1, #44]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
prfm pldl1keep, [x1, #512]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
add x1, x1, #48
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ins v0.s[0], w4
|
||||
|
||||
ins v1.s[0], w8
|
||||
ins v2.s[0], w12
|
||||
|
||||
ins v4.s[0], w20
|
||||
ins v5.s[0], w24
|
||||
|
||||
ins v0.s[1], w5
|
||||
ins v1.s[1], w9
|
||||
|
||||
ins v2.s[1], w13
|
||||
ins v4.s[1], w21
|
||||
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[2], w6
|
||||
|
||||
ins v1.s[2], w10
|
||||
ins v2.s[2], w14
|
||||
|
||||
ins v4.s[2], w22
|
||||
ins v5.s[2], w26
|
||||
|
||||
ins v0.s[3], w7
|
||||
ins v1.s[3], w11
|
||||
|
||||
ins v2.s[3], w15
|
||||
ins v4.s[3], w23
|
||||
ins v5.s[3], w27
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr x4, [x1]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr x20, [x2]
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1, #8]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr x21, [x2, #8]
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1, #16]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr x22, [x2, #16]
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1, #24]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr x23, [x2, #24]
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr x8, [x1, #32]
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
ldr x9, [x1, #40]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #384]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #448]
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
prfm pldl1keep, [x1, #512]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
add x1, x1, #48
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
add x2, x2, #32
|
||||
|
||||
|
||||
ins v0.d[0], x4
|
||||
ins v2.d[0], x8
|
||||
|
||||
ins v4.d[0], x20
|
||||
ins v5.d[0], x22
|
||||
|
||||
ins v0.d[1], x5
|
||||
ins v2.d[1], x9
|
||||
|
||||
ins v4.d[1], x21
|
||||
ins v1.d[0], x6
|
||||
|
||||
ins v1.d[1], x7
|
||||
|
||||
ins v5.d[1], x23
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s }}, [x1], #48
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
// mul a: v0, v1, v2, b: v4, v5
|
||||
// load a: d3/x23, d6/x26, d7/x27
|
||||
// load b: x4, x5, x6, x7
|
||||
|
||||
fmla v8.4s, v0.4s, v4.s[0]
|
||||
ldr d3, [x1], #8
|
||||
fmla v9.4s, v1.4s, v4.s[0]
|
||||
ldr x4, [x2], #8
|
||||
fmla v10.4s, v2.4s, v4.s[0]
|
||||
ldr x23, [x1], #8
|
||||
fmla v11.4s, v0.4s, v4.s[1]
|
||||
ldr x5, [x2], #8
|
||||
fmla v12.4s, v1.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v13.4s, v2.4s, v4.s[1]
|
||||
ldr x6, [x2], #8
|
||||
fmla v14.4s, v0.4s, v4.s[2]
|
||||
ldr x26, [x1], #8
|
||||
fmla v15.4s, v1.4s, v4.s[2]
|
||||
ldr x7, [x2], #8
|
||||
fmla v16.4s, v2.4s, v4.s[2]
|
||||
ldr d7, [x1], #8
|
||||
fmla v17.4s, v0.4s, v4.s[3]
|
||||
ldr x27, [x1], #8
|
||||
fmla v18.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v19.4s, v2.4s, v4.s[3]
|
||||
fmla v20.4s, v0.4s, v5.s[0]
|
||||
|
||||
// ins b: v4 <- x4/x5
|
||||
// ins a: d3/x23, d6/x26, d7/x27
|
||||
|
||||
ins v4.d[0], x4
|
||||
fmla v21.4s, v1.4s, v5.s[0]
|
||||
ins v4.d[1], x5
|
||||
fmla v22.4s, v2.4s, v5.s[0]
|
||||
fmla v23.4s, v0.4s, v5.s[1]
|
||||
|
||||
fmla v24.4s, v1.4s, v5.s[1]
|
||||
fmla v25.4s, v2.4s, v5.s[1]
|
||||
fmla v26.4s, v0.4s, v5.s[2]
|
||||
fmla v27.4s, v1.4s, v5.s[2]
|
||||
fmla v28.4s, v2.4s, v5.s[2]
|
||||
fmla v29.4s, v0.4s, v5.s[3]
|
||||
ins v3.d[1], x23
|
||||
fmla v30.4s, v1.4s, v5.s[3]
|
||||
ins v6.d[1], x26
|
||||
fmla v31.4s, v2.4s, v5.s[3]
|
||||
ins v7.d[1], x27
|
||||
|
||||
// mul a: v3, v6, v7, b: v4, v5
|
||||
// ins b, v5 <- x6, x7
|
||||
// load a: d0/x20, d1/x21, d2/x22
|
||||
// load b: x4, x5
|
||||
|
||||
fmla v8.4s, v3.4s, v4.s[0]
|
||||
ins v5.d[0], x6
|
||||
fmla v9.4s, v6.4s, v4.s[0]
|
||||
ins v5.d[1], x7
|
||||
fmla v10.4s, v7.4s, v4.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v11.4s, v3.4s, v4.s[1]
|
||||
ldr x4, [x2], #8
|
||||
fmla v12.4s, v6.4s, v4.s[1]
|
||||
ldr x20, [x1], #8
|
||||
fmla v13.4s, v7.4s, v4.s[1]
|
||||
ldr x5, [x2], #8
|
||||
fmla v14.4s, v3.4s, v4.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v15.4s, v6.4s, v4.s[2]
|
||||
ldr x6, [x2], #8
|
||||
fmla v16.4s, v7.4s, v4.s[2]
|
||||
ldr x21, [x1], #8
|
||||
fmla v17.4s, v3.4s, v4.s[3]
|
||||
ldr x7, [x2], #8
|
||||
|
||||
// load b: x6, x7
|
||||
fmla v18.4s, v6.4s, v4.s[3]
|
||||
ldr d2, [x1], #8
|
||||
fmla v19.4s, v7.4s, v4.s[3]
|
||||
ldr x22, [x1], #8
|
||||
fmla v20.4s, v3.4s, v5.s[0]
|
||||
fmla v21.4s, v6.4s, v5.s[0]
|
||||
fmla v22.4s, v7.4s, v5.s[0]
|
||||
fmla v23.4s, v3.4s, v5.s[1]
|
||||
fmla v24.4s, v6.4s, v5.s[1]
|
||||
fmla v25.4s, v7.4s, v5.s[1]
|
||||
|
||||
// ins a: d0/x20, d1/x21, d2/x22
|
||||
fmla v26.4s, v3.4s, v5.s[2]
|
||||
ins v0.d[1], x20
|
||||
fmla v27.4s, v6.4s, v5.s[2]
|
||||
ins v1.d[1], x21
|
||||
fmla v28.4s, v7.4s, v5.s[2]
|
||||
ins v2.d[1], x22
|
||||
|
||||
// ins b: v4 <- x4, x5
|
||||
fmla v29.4s, v3.4s, v5.s[3]
|
||||
ins v4.d[0], x4
|
||||
fmla v30.4s, v6.4s, v5.s[3]
|
||||
ins v4.d[1], x5
|
||||
fmla v31.4s, v7.4s, v5.s[3]
|
||||
|
||||
// ins b: v5 <- x6, x7
|
||||
ins v5.d[0], x6
|
||||
ins v5.d[1], x7
|
||||
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v8 v11 v14 v17 v20 v23 v26 v29
|
||||
// v9 v12 v15 v18 v21 v24 v27 v30
|
||||
// v10 v13 v16 v19 v22 v25 v28 v31
|
||||
|
||||
// no preservation for v0-v7:
|
||||
// packed A buffering (2x8 values): rotating over v0..v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_12x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_12x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s }, [ x1 ], #48
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop1/ldr_x_preload.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_12x8/packed_packed_loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 12 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 12 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8 ] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
{% for col in range(0, 8) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 3) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 3 + 8 + reg }}.4s, v{{ col * 3 + 8 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
{% for col in range(0, 8) %}
|
||||
{% for reg in range(0, 3) %}
|
||||
fmla v{{ col * 3 + 8 + reg }}.4s, v{{reg}}.4s, v{{ col // 4 + 4 }}.s[{{ col % 4 }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(0, 8) %}
|
||||
str q{{ col * 3 + 8 }}, [ x5 ]
|
||||
str q{{ col * 3 + 9 }}, [ x5, #16 ]
|
||||
str q{{ col * 3 + 10 }}, [ x5, #32 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
{% for col in range(0, 8) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 3) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 3 + 8 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x6, [x1, #8]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x7, [x1, #16]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x8, [x1, #24]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr x9, [x1, #32]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x1, #40]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr x11, [x1, #48]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x12, [x1, #56]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr x24, [x2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x25, [x2, #8]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
add x1, x1, #64
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
add x2, x2, #16
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v2.d[0], x9
|
||||
ins v1.d[0], x7
|
||||
ins v3.d[0], x11
|
||||
ins v4.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v2.d[1], x10
|
||||
ins v1.d[1], x8
|
||||
ins v3.d[1], x12
|
||||
ins v4.d[1], x25
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
ld1 {{ v4.4s }}, [ x2 ], #16
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
// mul a: v0, v1, v2, v3 b: v4
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9(d9/x9)
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4(d4/x9)
|
||||
|
||||
fmla v16.4s, v5.4s, v9.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v6.4s, v9.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v7.4s, v9.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v8.4s, v9.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v5.4s, v9.s[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v9.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v7.4s, v9.s[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.4s, v8.4s, v9.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v5.4s, v9.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.4s, v6.4s, v9.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v7.4s, v9.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.4s, v8.4s, v9.s[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.4s, v5.4s, v9.s[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.4s, v6.4s, v9.s[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.4s, v7.4s, v9.s[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.4s, v8.4s, v9.s[3]
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_16x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_16x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s }, [ x2 ], #16
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop1/cortex_a53.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_16x4/packed_packed_loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 16 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 16 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 4 + 16 + reg }}.4s, v{{ col * 4 + 16 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ]
|
||||
ld1 { v4.4s }, [ x3 ]
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
str q{{ col * 4 + 16 + 0 }}, [ x5 ]
|
||||
str q{{ col * 4 + 16 + 1 }}, [ x5, #16 ]
|
||||
str q{{ col * 4 + 16 + 2 }}, [ x5, #32 ]
|
||||
str q{{ col * 4 + 16 + 3 }}, [ x5, #48 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 4 + 16 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
// mul a: v0, v1, v2, v3, v4, v5 b: v7
|
||||
// load a: v5(d5/x5), v6(d6,x6), v7(d7,x7), v8(d8, x8)
|
||||
// load b: v9(d9/x9)
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d9, [x2], #8
|
||||
fmla v18.4s, v2.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[1]
|
||||
ldr d6, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v2.4s, v4.s[1]
|
||||
ldr d7, [x1], #8
|
||||
fmla v23.4s, v3.4s, v4.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v4.s[2]
|
||||
ldr d8, [x1], #8
|
||||
fmla v25.4s, v1.4s, v4.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v2.4s, v4.s[2]
|
||||
ins v5.d[1], x5
|
||||
fmla v27.4s, v3.4s, v4.s[2]
|
||||
ins v6.d[1], x6
|
||||
fmla v28.4s, v0.4s, v4.s[3]
|
||||
ins v7.d[1], x7
|
||||
fmla v29.4s, v1.4s, v4.s[3]
|
||||
ins v8.d[1], x8
|
||||
fmla v30.4s, v2.4s, v4.s[3]
|
||||
ins v9.d[1], x9
|
||||
fmla v31.4s, v3.4s, v4.s[3]
|
||||
|
||||
// mul a: v5, v6, v7, v8 b: v9
|
||||
// load a: v0(d0/x5), v1(d1,x6), v2(d2,x7), v3(d3, x8)
|
||||
// load b: v4(d4/x9)
|
||||
|
||||
fmla v16.4s, v5.4s, v9.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v6.4s, v9.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v7.4s, v9.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v8.4s, v9.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v20.4s, v5.4s, v9.s[1]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v9.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v22.4s, v7.4s, v9.s[1]
|
||||
ldr d2, [x1], #8
|
||||
fmla v23.4s, v8.4s, v9.s[1]
|
||||
ldr x7, [x1], #8
|
||||
|
||||
fmla v24.4s, v5.4s, v9.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v25.4s, v6.4s, v9.s[2]
|
||||
ldr x8, [x1], #8
|
||||
fmla v26.4s, v7.4s, v9.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v27.4s, v8.4s, v9.s[2]
|
||||
ins v1.d[1], x6
|
||||
fmla v28.4s, v5.4s, v9.s[3]
|
||||
ins v2.d[1], x7
|
||||
fmla v29.4s, v6.4s, v9.s[3]
|
||||
ins v3.d[1], x8
|
||||
fmla v30.4s, v7.4s, v9.s[3]
|
||||
ins v4.d[1], x9
|
||||
fmla v31.4s, v8.4s, v9.s[3]
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
ldr x4, [x1]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
ldr x5, [x1, #8]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
ldr x6, [x1, #16]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
ldr x7, [x1, #24]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
ldr x8, [x1, #32]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
ldr x9, [x1, #40]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
ldr x10, [x1, #48]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
ldr x11, [x1, #56]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
ldr x12, [x1, #64]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
ldr x13, [x1, #72]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
ldr x14, [x1, #80]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
ldr x15, [x1, #88]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
ldr x20, [x2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
ldr x21, [x2, #8]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
add x1, x1, #96
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
add x2, x2, #16
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
prfm pldl1keep, [x1, #384]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
prfm pldl1keep, [x1, #448]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
prfm pldl1keep, [x2, #448]
|
||||
|
||||
ins v0.d[0], x4
|
||||
ins v1.d[0], x6
|
||||
ins v2.d[0], x8
|
||||
ins v3.d[0], x10
|
||||
ins v4.d[0], x12
|
||||
ins v5.d[0], x14
|
||||
ins v7.d[0], x20
|
||||
|
||||
ins v0.d[1], x5
|
||||
ins v1.d[1], x7
|
||||
ins v2.d[1], x9
|
||||
ins v3.d[1], x11
|
||||
ins v4.d[1], x13
|
||||
ins v5.d[1], x15
|
||||
ins v7.d[1], x21
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
ldr d0, [x1], #8
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
ldr x20, [x2], #8
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
ldr x21, [x2], #8
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
ldr d1, [x1], #8
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
ldr x7, [x1], #8
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
ldr d2, [x1], #8
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
ldr x9, [x1], #8
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
ins v0.d[1], x5
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
ins v1.d[1], x7
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
ldr d3, [x1], #8
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
ldr x11, [x1], #8
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
ins v2.d[1], x9
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
ldr d4, [x1], #8
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
ldr x14, [x1], #8
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
ins v7.d[0], x20
|
||||
ins v7.d[1], x21
|
||||
|
||||
ins v5.d[0], x14
|
||||
ins v5.d[1], x15
|
||||
|
||||
ins v3.d[1], x11
|
||||
ins v4.d[1], x13
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
ld1 {{ v4.4s, v5.4s }}, [ x1 ], #32
|
||||
ld1 {{ v7.4s }}, [ x2 ], #16
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// x20..x27 are used, callee-preserved
|
||||
|
||||
// C tile regs: v8 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// v8 is used, d8 (lower half) must preserved
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_24x4_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_24x4_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s }, [ x1 ], #32
|
||||
ld1 { v7.4s }, [ x2 ], #16
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/cortex_a53.S.raw" %}
|
||||
{% elif core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_24x4/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 24 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 24 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 6) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 6 + 8 + reg }}.4s, v{{ col * 6 + 8 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ], #64
|
||||
ld1 { v7.4s }, [ x3 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ]
|
||||
|
||||
fmla v8.4s, v0.4s, v7.s[0]
|
||||
fmla v9.4s, v1.4s, v7.s[0]
|
||||
fmla v10.4s, v2.4s, v7.s[0]
|
||||
fmla v11.4s, v3.4s, v7.s[0]
|
||||
fmla v12.4s, v4.4s, v7.s[0]
|
||||
fmla v13.4s, v5.4s, v7.s[0]
|
||||
|
||||
fmla v14.4s, v0.4s, v7.s[1]
|
||||
fmla v15.4s, v1.4s, v7.s[1]
|
||||
fmla v16.4s, v2.4s, v7.s[1]
|
||||
fmla v17.4s, v3.4s, v7.s[1]
|
||||
fmla v18.4s, v4.4s, v7.s[1]
|
||||
fmla v19.4s, v5.4s, v7.s[1]
|
||||
|
||||
fmla v20.4s, v0.4s, v7.s[2]
|
||||
fmla v21.4s, v1.4s, v7.s[2]
|
||||
fmla v22.4s, v2.4s, v7.s[2]
|
||||
fmla v23.4s, v3.4s, v7.s[2]
|
||||
fmla v24.4s, v4.4s, v7.s[2]
|
||||
fmla v25.4s, v5.4s, v7.s[2]
|
||||
|
||||
fmla v26.4s, v0.4s, v7.s[3]
|
||||
fmla v27.4s, v1.4s, v7.s[3]
|
||||
fmla v28.4s, v2.4s, v7.s[3]
|
||||
fmla v29.4s, v3.4s, v7.s[3]
|
||||
fmla v30.4s, v4.4s, v7.s[3]
|
||||
fmla v31.4s, v5.4s, v7.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
str q{{ col * 6 + 8 + 0 }}, [ x5 ]
|
||||
str q{{ col * 6 + 8 + 1 }}, [ x5, #16 ]
|
||||
str q{{ col * 6 + 8 + 2 }}, [ x5, #32 ]
|
||||
str q{{ col * 6 + 8 + 3 }}, [ x5, #48 ]
|
||||
str q{{ col * 6 + 8 + 4 }}, [ x5, #64 ]
|
||||
str q{{ col * 6 + 8 + 5 }}, [ x5, #80 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(0, 4) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 6) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 6 + 8 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_32x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_32x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .q4f16se
|
||||
cmp x4, #2
|
||||
beq .q4f32se
|
||||
cmp x4, #3
|
||||
beq .f16f16
|
||||
cmp x4, #4
|
||||
beq .f32f16
|
||||
cmp x4, #5
|
||||
beq .f16f32
|
||||
|
||||
sub x3, x3, #1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
ld1 { v8.s }[0], [ x2 ], #4
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 8
|
||||
.q40f16_const:
|
||||
.byte 0xc8, 0xc7, 0xc6, 0xc5, 0xc4, 0xc2, 0xc0, 0xbc
|
||||
.byte 0x00, 0x3c, 0x40, 0x42, 0x44, 0x45, 0x46, 0x47
|
||||
|
||||
.q4f16se:
|
||||
adr x4, .q40f16_const
|
||||
movi v15.16b, 15
|
||||
ld1 {v13.16b}, [ x4 ]
|
||||
eor v12.16b, v12.16b, v12.16b
|
||||
|
||||
.q4f16se_outerloop:
|
||||
{% for i in range(0, 8) %}
|
||||
eor v{{ i + 16 }}.16b, v{{ i + 16 }}.16b, v{{ i + 16 }}.16b
|
||||
{% endfor %}
|
||||
mov x4, #32
|
||||
|
||||
.p2align 4
|
||||
.q4f16se_innerloop:
|
||||
ld1 { v10.16b }, [ x1 ], #16
|
||||
ld1 { v11.h }[0], [ x2 ], #2
|
||||
|
||||
and v9.16b, v10.16b, v15.16b
|
||||
ushr v10.16b, v10.16b, 4
|
||||
|
||||
tbl v9.16b, { v13.16b }, v9.16b
|
||||
tbl v10.16b, { v13.16b }, v10.16b
|
||||
|
||||
zip1 v0.16b, v12.16b, v9.16b
|
||||
zip2 v2.16b, v12.16b, v9.16b
|
||||
zip1 v4.16b, v12.16b, v10.16b
|
||||
zip2 v6.16b, v12.16b, v10.16b
|
||||
|
||||
fcvtl v11.4s, v11.4h
|
||||
|
||||
fcvtl2 v1.4s, v0.8h
|
||||
fcvtl2 v3.4s, v2.8h
|
||||
fcvtl2 v5.4s, v4.8h
|
||||
fcvtl2 v7.4s, v6.8h
|
||||
fcvtl v0.4s, v0.4h
|
||||
fcvtl v2.4s, v2.4h
|
||||
fcvtl v4.4s, v4.4h
|
||||
fcvtl v6.4s, v6.4h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 16 }}.4s, v{{i}}.4s, v11.s[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x4, x4, #1
|
||||
bne .q4f16se_innerloop
|
||||
|
||||
// scales
|
||||
ld1 { v0.8h-v3.8h }, [ x1 ], #64
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v7.4s, v1.8h
|
||||
fcvtl v8.4s, v2.4h
|
||||
fcvtl2 v9.4s, v2.8h
|
||||
fcvtl v10.4s, v3.4h
|
||||
fcvtl2 v11.4s, v3.8h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 24 }}.4s, v{{ i + 4 }}.4s, v{{ i + 16 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #32
|
||||
bne .q4f16se_outerloop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q4f32se:
|
||||
adr x4, .q40f16_const
|
||||
movi v15.16b, 15
|
||||
ld1 {v13.16b}, [ x4 ]
|
||||
eor v12.16b, v12.16b, v12.16b
|
||||
|
||||
.q4f32se_outerloop:
|
||||
{% for i in range(0, 8) %}
|
||||
eor v{{ i + 16 }}.16b, v{{ i + 16 }}.16b, v{{ i + 16 }}.16b
|
||||
{% endfor %}
|
||||
mov x4, #32
|
||||
|
||||
.p2align 4
|
||||
.q4f32se_innerloop:
|
||||
ld1 { v10.16b }, [ x1 ], #16
|
||||
ld1 { v11.s }[0], [ x2 ], #4
|
||||
|
||||
and v9.16b, v10.16b, v15.16b
|
||||
ushr v10.16b, v10.16b, 4
|
||||
|
||||
tbl v9.16b, { v13.16b }, v9.16b
|
||||
tbl v10.16b, { v13.16b }, v10.16b
|
||||
|
||||
zip1 v0.16b, v12.16b, v9.16b
|
||||
zip2 v2.16b, v12.16b, v9.16b
|
||||
zip1 v4.16b, v12.16b, v10.16b
|
||||
zip2 v6.16b, v12.16b, v10.16b
|
||||
|
||||
fcvtl2 v1.4s, v0.8h
|
||||
fcvtl2 v3.4s, v2.8h
|
||||
fcvtl2 v5.4s, v4.8h
|
||||
fcvtl2 v7.4s, v6.8h
|
||||
fcvtl v0.4s, v0.4h
|
||||
fcvtl v2.4s, v2.4h
|
||||
fcvtl v4.4s, v4.4h
|
||||
fcvtl v6.4s, v6.4h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 16 }}.4s, v{{i}}.4s, v11.s[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x4, x4, #1
|
||||
bne .q4f32se_innerloop
|
||||
|
||||
// scales
|
||||
ld1 { v0.8h-v3.8h }, [ x1 ], #64
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v7.4s, v1.8h
|
||||
fcvtl v8.4s, v2.4h
|
||||
fcvtl2 v9.4s, v2.8h
|
||||
fcvtl v10.4s, v3.4h
|
||||
fcvtl2 v11.4s, v3.8h
|
||||
|
||||
{% for i in range(0, 8) %}
|
||||
fmla v{{ i + 24 }}.4s, v{{ i + 4 }}.4s, v{{ i + 16 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #32
|
||||
bne .q4f32se_outerloop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f16:
|
||||
sub x3, x3, #1
|
||||
.f16f16_loop:
|
||||
ld1 { v9.h }[0], [ x2 ], #2
|
||||
ld1 { v10.8h-v13.8h }, [ x1 ], #64
|
||||
|
||||
fcvtl v8.4s, v9.4h
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtl v{{ reg * 2 }}.4s, v{{ reg + 10 }}.4h
|
||||
fcvtl2 v{{ reg * 2 + 1 }}.4s, v{{ reg + 10 }}.8h
|
||||
{% endfor %}
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .f16f16_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f32f16:
|
||||
sub x3, x3, #1
|
||||
.f32f16_loop:
|
||||
ld1 { v9.h }[0], [ x2 ], #2
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
fcvtl v8.4s, v9.4h
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .f32f16_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f32:
|
||||
sub x3, x3, #1
|
||||
.f16f32_loop:
|
||||
ld1 { v8.s }[0], [ x2 ], #4
|
||||
ld1 { v10.8h-v13.8h }, [ x1 ], #64
|
||||
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtl v{{ reg * 2 }}.4s, v{{ reg + 10 }}.4h
|
||||
fcvtl2 v{{ reg * 2 + 1 }}.4s, v{{ reg + 10 }}.8h
|
||||
{% endfor %}
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
subs x3, x3, #1
|
||||
bge .f16f32_loop
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 24 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #4
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.4s-v3.4s}, [x5], #64
|
||||
ld1 {v4.4s-v7.4s}, [x5], #64
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
fadd v{{ r + 24 }}.4s, v{{ r + 24 }}.4s, v{{r}}.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.s}[0], [ x3 ]
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x2 ], #64
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #2
|
||||
beq .store_f16
|
||||
|
||||
cmp x6, #4
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in range(24, 32) %}
|
||||
st1 { v{{reg}}.4s }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_f16:
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtn v{{reg}}.4h, v{{ reg * 2 + 24 }}.4s
|
||||
fcvtn2 v{{reg}}.8h, v{{ reg * 2 + 25 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig_f16
|
||||
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{reg}}.h }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig_f16:
|
||||
|
||||
{% for reg in range(0, 4) %}
|
||||
st1 { v{{reg}}.8h }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// v8..v15 are callee-preserved
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.global {{G}}arm64simd_mmm_f32_32x3_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_32x3_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .f32f16
|
||||
cmp x4, #2
|
||||
beq .f16f32
|
||||
cmp x4, #3
|
||||
beq .f16f16
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
ld1 { v7.4s }, [ x2 ]
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s }, [ x1 ], #48
|
||||
add x2, x2, #12
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s }, [ x1 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.4s, v{{row}}.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f32f16:
|
||||
ld1 { v7.4h }, [ x2 ]
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s }, [ x1 ], #48
|
||||
fcvtl v7.4s, v7.4h
|
||||
add x2, x2, #6
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s }, [ x1 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.4s, v{{row}}.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .f32f16
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f32:
|
||||
ld1 { v7.4s }, [ x2 ]
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x1 ], #64
|
||||
add x2, x2, #12
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v0.4s, v1.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 9 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 10 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 11 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
fcvtl v4.4s, v2.4h
|
||||
fcvtl2 v5.4s, v2.8h
|
||||
fcvtl v6.4s, v3.4h
|
||||
fcvtl2 v1.4s, v3.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 12 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 13 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 14 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 15 }}.4s, v1.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .f16f32
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.f16f16:
|
||||
ld1 { v7.4h }, [ x2 ]
|
||||
ld1 { v0.8h, v1.8h, v2.8h, v3.8h }, [ x1 ], #64
|
||||
add x2, x2, #6
|
||||
|
||||
fcvtl v7.4s, v7.4h
|
||||
|
||||
fcvtl v4.4s, v0.4h
|
||||
fcvtl2 v5.4s, v0.8h
|
||||
fcvtl v6.4s, v1.4h
|
||||
fcvtl2 v0.4s, v1.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 9 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 10 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 11 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
fcvtl v4.4s, v2.4h
|
||||
fcvtl2 v5.4s, v2.8h
|
||||
fcvtl v6.4s, v3.4h
|
||||
fcvtl2 v1.4s, v3.8h
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 12 }}.4s, v4.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 13 }}.4s, v5.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 14 }}.4s, v6.4s, v7.s[{{ col }}]
|
||||
fmla v{{ col * 8 + 15 }}.4s, v1.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .f16f16
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 32 %}{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 8 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 8) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 8 + 8 + reg }}.4s, v{{ col * 8 + 8 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldp x2, x3, [x0, #8]
|
||||
|
||||
ld1 { v7.d }[0], [ x3 ], #8
|
||||
ld1 { v7.s }[2], [ x3 ], #4
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s }, [ x2 ], #48
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s }, [ x2 ], #16
|
||||
|
||||
{% for row in range(1, 7) %}
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 8 + row }}.4s, v{{row}}.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
fmla v{{ col * 8 + 15 }}.4s, v0.4s, v7.s[{{ col }}]
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #2
|
||||
beq .store_f16
|
||||
|
||||
cmp x6, #4
|
||||
beq .store_strides_contig
|
||||
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 8) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 8 + 8 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
mov x4, x5
|
||||
{% for r in range(0, 8) %}
|
||||
st1 { v{{ col * 8 + 8 + r }}.4s }, [ x4 ], 16
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_f16:
|
||||
|
||||
cmp x6, #2
|
||||
beq .store_strides_contig_f16
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtn v{{reg}}.4h, v{{ (col * 4 + reg) * 2 + 8 }}.4s
|
||||
fcvtn2 v{{reg}}.8h, v{{ (col * 4 + reg) * 2 + 9 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
{% for lane in range(0, 8) %}
|
||||
st1 { v{{reg}}.h }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
|
||||
{% endfor %}
|
||||
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig_f16:
|
||||
|
||||
{% for col in range(0, 3) %}
|
||||
{% for reg in range(0, 4) %}
|
||||
fcvtn v{{reg}}.4h, v{{ (col * 4 + reg) * 2 + 8 }}.4s
|
||||
fcvtn2 v{{reg}}.8h, v{{ (col * 4 + reg) * 2 + 9 }}.4s
|
||||
{% endfor %}
|
||||
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 4) %}
|
||||
st1 { v{{reg}}.4s }, [ x4 ], #16
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x5, [x1, #128]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr x6, [x1, #136]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x7, [x1, #144]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr x9, [x1, #152]
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [ x1 ], #64
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x10, [x1, #96]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr x11, [x1, #104]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x12, [x1, #112]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr x13, [x1, #120]
|
||||
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [ x1 ]
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
ldr x14, [x1, #128]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
ldr x15, [x1, #136]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
ldr x20, [x1, #144]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
ldr x21, [x1, #152]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
ldr x22, [x1, #160]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
ldr x23, [x1, #168]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
ldr x24, [x1, #176]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
ldr x25, [x1, #184]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v1.d[0], x7
|
||||
ins v2.d[0], x10
|
||||
ins v3.d[0], x12
|
||||
ins v4.d[0], x14
|
||||
ins v5.d[0], x20
|
||||
ins v6.d[0], x22
|
||||
ins v7.d[0], x24
|
||||
|
||||
ins v0.d[1], x6
|
||||
ins v1.d[1], x9
|
||||
ins v2.d[1], x11
|
||||
ins v3.d[1], x13
|
||||
ins v4.d[1], x15
|
||||
ins v5.d[1], x21
|
||||
ins v6.d[1], x23
|
||||
ins v7.d[1], x25
|
||||
|
||||
add x1, x1, #192
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
ld1 {{ v9.4s }}, [ x1 ], #16
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
|
||||
fmla v31.4s, v9.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr x8, [x2], #8
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr d12, [x1], #8
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr x12, [x1], #8
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr d14, [x1], #8
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr x14, [x1], #8
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr d15, [x1], #8
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
ldr x15, [x1], #8
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.d[1], x8
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
ins v12.d[1], x12
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
ins v13.d[1], x13
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
ins v14.d[1], x14
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
ins v15.d[1], x15
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
ldr d13, [x1], #8
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
ldr x13, [x1], #8
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
ldr d14, [x1], #8
|
||||
|
||||
fmla v16.4s, v1.4s, v8.s[2]
|
||||
ldr x14, [x1], #8
|
||||
fmla v17.4s, v2.4s, v8.s[2]
|
||||
ldr d15, [x1], #8
|
||||
fmla v18.4s, v3.4s, v8.s[2]
|
||||
ldr x15, [x1], #8
|
||||
fmla v19.4s, v4.4s, v8.s[2]
|
||||
|
||||
ld1 {{ v0.4s }}, [x1], #16
|
||||
|
||||
fmla v20.4s, v5.4s, v8.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v6.4s, v8.s[2]
|
||||
ldr x10, [x1], #8
|
||||
|
||||
fmla v22.4s, v7.4s, v8.s[2]
|
||||
|
||||
fmla v23.4s, v9.4s, v8.s[2]
|
||||
ins v13.d[1], x13
|
||||
fmla v24.4s, v10.4s, v8.s[2]
|
||||
ins v14.d[1], x14
|
||||
fmla v25.4s, v11.4s, v8.s[2]
|
||||
ins v15.d[1], x15
|
||||
|
||||
fmla v26.4s, v12.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
fmla v27.4s, v13.4s, v8.s[2]
|
||||
ins v1.d[1], x10
|
||||
fmla v28.4s, v14.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
fmla v29.4s, v15.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
fmla v30.4s, v0.4s, v8.s[2]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
fmla v31.4s, v1.4s, v8.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ins v8.s[0], v8.s[3]
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
// load a: v9, v10, v11, v12, v13, v14, v15
|
||||
// load a: v0, v1, v2, v3, v4, v4, v6, v7
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v2.4s, v3.4s, v4.4s, v5.4s }}, [x1], #64
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v6.4s, v7.4s }}, [x1], #32
|
||||
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
|
||||
ld1 {{ v9.4s, v10.4s, v11.4s, v12.4s }}, [x1], #64
|
||||
ld1 {{ v13.4s, v14.4s, v15.4s }}, [x1], #48
|
||||
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
fmla v16.4s, v1.4s, v8.s[0]
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
fmla v17.4s, v2.4s, v8.s[0]
|
||||
fmla v18.4s, v3.4s, v8.s[0]
|
||||
fmla v19.4s, v4.4s, v8.s[0]
|
||||
|
||||
fmla v20.4s, v5.4s, v8.s[0]
|
||||
fmla v21.4s, v6.4s, v8.s[0]
|
||||
fmla v22.4s, v7.4s, v8.s[0]
|
||||
fmla v23.4s, v9.4s, v8.s[0]
|
||||
|
||||
fmla v24.4s, v10.4s, v8.s[0]
|
||||
fmla v25.4s, v11.4s, v8.s[0]
|
||||
fmla v26.4s, v12.4s, v8.s[0]
|
||||
fmla v27.4s, v13.4s, v8.s[0]
|
||||
fmla v28.4s, v14.4s, v8.s[0]
|
||||
fmla v29.4s, v15.4s, v8.s[0]
|
||||
fmla v30.4s, v0.4s, v8.s[0]
|
||||
fmla v31.4s, v1.4s, v8.s[0]
|
||||
ld1 {{ v8.s }}[0], [ x2 ], #4
|
||||
|
||||
ld1 {{ v0.4s, v1.4s, v2.4s, v3.4s }}, [x1], #64
|
||||
ld1 {{ v4.4s, v5.4s, v6.4s, v7.4s }}, [x1], #64
|
||||
|
||||
prfm pldl1keep, [x1, #1024]
|
||||
prfm pldl1keep, [x1, #1088]
|
||||
prfm pldl1keep, [x1, #1152]
|
||||
prfm pldl1keep, [x1, #1216]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_64x1_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_64x1_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
sub x3, x3, #1
|
||||
|
||||
|
||||
ld1 { v8.s }[0], [ x2 ], #4
|
||||
ld1 { v0.4s, v1.4s, v2.4s, v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [ x1 ], #64
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop1/cortex_a53.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a53" %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% elif core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_64x1/loop2/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
cmp x3, #0
|
||||
beq .packed_packed_loop_1_last
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
// last loop can't read beyond actual input as it's likely not packed and padded
|
||||
.packed_packed_loop_1_last:
|
||||
ld1 { v9.4s, v10.4s, v11.4s, v12.4s }, [x1], #64
|
||||
ld1 { v13.4s, v14.4s, v15.4s }, [x1], #48
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ld1 { v0.4s }, [ x1 ]
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
|
||||
fmla v24.4s, v9.4s, v8.s[0]
|
||||
fmla v25.4s, v10.4s, v8.s[0]
|
||||
fmla v26.4s, v11.4s, v8.s[0]
|
||||
fmla v27.4s, v12.4s, v8.s[0]
|
||||
fmla v28.4s, v13.4s, v8.s[0]
|
||||
fmla v29.4s, v14.4s, v8.s[0]
|
||||
fmla v30.4s, v15.4s, v8.s[0]
|
||||
fmla v31.4s, v0.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
cmp x6, #4
|
||||
beq .do_per_row_add
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.do_per_row_add:
|
||||
ld1 {v0.4s-v3.4s}, [x5], #64
|
||||
ld1 {v4.4s-v7.4s}, [x5], #64
|
||||
ld1 {v8.4s-v11.4s}, [x5], #64
|
||||
ld1 {v12.4s-v15.4s}, [x5], #64
|
||||
|
||||
{% for r in range(0, 16) %}
|
||||
fadd v{{ r + 16 }}.4s, v{{ r + 16 }}.4s, v{{r}}.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x3, [x0, #16]
|
||||
ldr x2, [x0, #8]
|
||||
|
||||
ld1 {v8.s}[0], [ x3 ]
|
||||
|
||||
{% for r in range(0, 8) %}
|
||||
ldr q{{r}}, [x2], #16
|
||||
{% endfor %}
|
||||
|
||||
fmla v16.4s, v0.4s, v8.s[0]
|
||||
ldr q0, [x2], #16
|
||||
fmla v17.4s, v1.4s, v8.s[0]
|
||||
ldr q1, [x2], #16
|
||||
fmla v18.4s, v2.4s, v8.s[0]
|
||||
ldr q2, [x2], #16
|
||||
fmla v19.4s, v3.4s, v8.s[0]
|
||||
ldr q3, [x2], #16
|
||||
fmla v20.4s, v4.4s, v8.s[0]
|
||||
ldr q4, [x2], #16
|
||||
fmla v21.4s, v5.4s, v8.s[0]
|
||||
ldr q5, [x2], #16
|
||||
fmla v22.4s, v6.4s, v8.s[0]
|
||||
ldr q6, [x2], #16
|
||||
fmla v23.4s, v7.4s, v8.s[0]
|
||||
ldr q7, [x2], #16
|
||||
|
||||
fmla v24.4s, v0.4s, v8.s[0]
|
||||
fmla v25.4s, v1.4s, v8.s[0]
|
||||
fmla v26.4s, v2.4s, v8.s[0]
|
||||
fmla v27.4s, v3.4s, v8.s[0]
|
||||
fmla v28.4s, v4.4s, v8.s[0]
|
||||
fmla v29.4s, v5.4s, v8.s[0]
|
||||
fmla v30.4s, v6.4s, v8.s[0]
|
||||
fmla v31.4s, v7.4s, v8.s[0]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc$
|
||||
|
||||
cmp x6, #4
|
||||
beq .store_strides_contig
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_contig:
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
st1 { v{{reg}}.4s }, [ x5 ], #16
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
ld1 {{ v2.4s, v3.4s }}, [x1], #32
|
||||
ld1 {{ v6.4s, v7.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
and v0.16b, v2.16b, v2.16b
|
||||
and v1.16b, v3.16b, v3.16b
|
||||
and v4.16b, v6.16b, v6.16b
|
||||
and v5.16b, v7.16b, v7.16b
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v20.4s, v1.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr w22, [x2], #4
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr w8, [x1], #4
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr w23, [x2], #4
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
ldr w9, [x1], #4
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
ldr w24, [x2], #4
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
ldr w10, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
ldr w25, [x2], #4
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ldr w11, [x1], #4
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ldr w26, [x2], #4
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ldr w12, [x1], #4
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ldr w27, [x2], #4
|
||||
|
||||
ins v0.s[0], w5
|
||||
ins v4.s[0], w20
|
||||
ins v1.s[0], w9
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[2], w7
|
||||
ins v4.s[2], w22
|
||||
ins v1.s[2], w11
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[1], w6
|
||||
ins v4.s[1], w21
|
||||
ins v1.s[1], w10
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[3], w8
|
||||
ins v4.s[3], w23
|
||||
ins v1.s[3], w12
|
||||
ins v5.s[3], w27
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr w5, [x1], #4
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr w20, [x2], #4
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr w6, [x1], #4
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr w21, [x2], #4
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr w7, [x1], #4
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr w22, [x2], #4
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr w8, [x1], #4
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr w23, [x2], #4
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
ldr w9, [x1], #4
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
ldr w24, [x2], #4
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
ldr w10, [x1], #4
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
ldr w25, [x2], #4
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ldr w11, [x1], #4
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ldr w26, [x2], #4
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ldr w12, [x1], #4
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ldr w27, [x2], #4
|
||||
|
||||
prfm pldl1keep, [x1, #256]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
|
||||
ins v0.s[0], w5
|
||||
ins v4.s[0], w20
|
||||
ins v1.s[0], w9
|
||||
ins v5.s[0], w24
|
||||
ins v0.s[2], w7
|
||||
ins v4.s[2], w22
|
||||
ins v1.s[2], w11
|
||||
ins v5.s[2], w26
|
||||
ins v0.s[1], w6
|
||||
ins v4.s[1], w21
|
||||
ins v1.s[1], w10
|
||||
ins v5.s[1], w25
|
||||
ins v0.s[3], w8
|
||||
ins v4.s[3], w23
|
||||
ins v1.s[3], w12
|
||||
ins v5.s[3], w27
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr x11, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x8, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x12, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ins v2.d[0], x5
|
||||
ins v6.d[0], x9
|
||||
ins v3.d[0], x7
|
||||
ins v7.d[0], x11
|
||||
ins v2.d[1], x6
|
||||
ins v6.d[1], x10
|
||||
ins v3.d[1], x8
|
||||
ins v7.d[1], x12
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr x5, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr x9, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x6, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x10, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr x7, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr x11, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x8, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x12, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #256]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
prfm pldl1keep, [x1, #320]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #384]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
prfm pldl1keep, [x1, #448]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
prfm pldl1keep, [x2, #256]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
prfm pldl1keep, [x2, #320]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #384]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
prfm pldl1keep, [x2, #448]
|
||||
|
||||
ins v0.d[0], x5
|
||||
ins v4.d[0], x9
|
||||
ins v1.d[0], x7
|
||||
ins v5.d[0], x11
|
||||
ins v0.d[1], x6
|
||||
ins v4.d[1], x10
|
||||
ins v1.d[1], x8
|
||||
ins v5.d[1], x12
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
ld1 {{ v2.4s, v3.4s }}, [x1], #32
|
||||
ld1 {{ v6.4s, v7.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
ld1 {{ v0.4s, v1.4s }}, [x1], #32
|
||||
ld1 {{ v4.4s, v5.4s }}, [x2], #32
|
||||
|
||||
fmla v16.4s, v2.4s, v6.s[0]
|
||||
fmla v17.4s, v3.4s, v6.s[0]
|
||||
fmla v18.4s, v2.4s, v6.s[1]
|
||||
fmla v19.4s, v3.4s, v6.s[1]
|
||||
fmla v20.4s, v2.4s, v6.s[2]
|
||||
fmla v21.4s, v3.4s, v6.s[2]
|
||||
fmla v22.4s, v2.4s, v6.s[3]
|
||||
fmla v23.4s, v3.4s, v6.s[3]
|
||||
|
||||
fmla v24.4s, v2.4s, v7.s[0]
|
||||
fmla v25.4s, v3.4s, v7.s[0]
|
||||
fmla v26.4s, v2.4s, v7.s[1]
|
||||
fmla v27.4s, v3.4s, v7.s[1]
|
||||
fmla v28.4s, v2.4s, v7.s[2]
|
||||
fmla v29.4s, v3.4s, v7.s[2]
|
||||
fmla v30.4s, v2.4s, v7.s[3]
|
||||
fmla v31.4s, v3.4s, v7.s[3]
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
ldr d2, [x1], #8
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
ldr d6, [x2], #8
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
ldr d3, [x1], #8
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
ldr d7, [x2], #8
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
ldr x8, [x2], #8
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
ins v2.d[1], x5
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
ins v6.d[1], x7
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
ins v3.d[1], x6
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
ins v7.d[1], x8
|
||||
|
||||
fmla v16.4s, v2.4s, v6.s[0]
|
||||
ldr d0, [x1], #8
|
||||
fmla v17.4s, v3.4s, v6.s[0]
|
||||
ldr d4, [x2], #8
|
||||
fmla v18.4s, v2.4s, v6.s[1]
|
||||
ldr x5, [x1], #8
|
||||
fmla v19.4s, v3.4s, v6.s[1]
|
||||
ldr x7, [x2], #8
|
||||
fmla v20.4s, v2.4s, v6.s[2]
|
||||
ldr d1, [x1], #8
|
||||
fmla v21.4s, v3.4s, v6.s[2]
|
||||
ldr d5, [x2], #8
|
||||
fmla v22.4s, v2.4s, v6.s[3]
|
||||
ldr x6, [x1], #8
|
||||
fmla v23.4s, v3.4s, v6.s[3]
|
||||
ldr x8, [x2], #8
|
||||
|
||||
fmla v24.4s, v2.4s, v7.s[0]
|
||||
fmla v25.4s, v3.4s, v7.s[0]
|
||||
fmla v26.4s, v2.4s, v7.s[1]
|
||||
fmla v27.4s, v3.4s, v7.s[1]
|
||||
fmla v28.4s, v2.4s, v7.s[2]
|
||||
ins v0.d[1], x5
|
||||
fmla v29.4s, v3.4s, v7.s[2]
|
||||
ins v4.d[1], x7
|
||||
fmla v30.4s, v2.4s, v7.s[3]
|
||||
ins v1.d[1], x6
|
||||
fmla v31.4s, v3.4s, v7.s[3]
|
||||
ins v5.d[1], x8
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs: v16 to v31, (scratch)
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// v0-v7 (scratch registers)
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_f32_8x8_{{core}}_{{suffix}}
|
||||
{{G}}arm64simd_mmm_f32_8x8_{{core}}_{{suffix}}:
|
||||
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldr x2, [x0, #24] // b
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.packed_packed:
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
{% set packed_packed_loop1 %}
|
||||
{% if core == "a53" %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop1/ldr_x_preload.S.raw" %}
|
||||
{% else %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop1/naive.S.raw" %}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
{% set packed_packed_loop2 %}
|
||||
{% if core == "a55" %}
|
||||
{% include "arm64simd_mmm_f32_8x8/packed_packed_loop2/cortex_a55.S.raw" %}
|
||||
{% else %}
|
||||
{{ packed_packed_loop1 }}
|
||||
{{ packed_packed_loop1 }}
|
||||
{% endif %}
|
||||
{% endset %}
|
||||
|
||||
cmp x3, #4
|
||||
blt .packed_packed_loop_1
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_4:
|
||||
{{ packed_packed_loop2 }}
|
||||
{{ packed_packed_loop2 }}
|
||||
|
||||
sub x3, x3, #4
|
||||
cmp x3, #4
|
||||
bge .packed_packed_loop_4
|
||||
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
.p2align 4
|
||||
.packed_packed_loop_1:
|
||||
{{ packed_packed_loop1 }}
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_f32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
fadd v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ], #32
|
||||
|
||||
fmla v16.4s, v0.4s, v4.s[0]
|
||||
fmla v17.4s, v1.4s, v4.s[0]
|
||||
fmla v18.4s, v0.4s, v4.s[1]
|
||||
fmla v19.4s, v1.4s, v4.s[1]
|
||||
fmla v20.4s, v0.4s, v4.s[2]
|
||||
fmla v21.4s, v1.4s, v4.s[2]
|
||||
fmla v22.4s, v0.4s, v4.s[3]
|
||||
fmla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
fmla v24.4s, v0.4s, v5.s[0]
|
||||
fmla v25.4s, v1.4s, v5.s[0]
|
||||
fmla v26.4s, v0.4s, v5.s[1]
|
||||
fmla v27.4s, v1.4s, v5.s[1]
|
||||
fmla v28.4s, v0.4s, v5.s[2]
|
||||
fmla v29.4s, v1.4s, v5.s[2]
|
||||
fmla v30.4s, v0.4s, v5.s[3]
|
||||
fmla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x6, #4
|
||||
bne .store_strides_generic
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
str q{{ col * 2 }}, [ x5 ]
|
||||
str q{{ col * 2 + 1 }}, [ x5, #16 ]
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_generic:
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col(label="per_col_min", op="fmin", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_max", op="fmax", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_mul", op="fmul", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_add", op="fadd", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub", op="fsub", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub_flipped", op="fsub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row(label="per_row_min", op="fmin", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_max", op="fmax", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_mul", op="fmul", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_add", op="fadd", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub", op="fsub", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub_flipped", op="fsub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar(label="scalar_min", op="fmin", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_max", op="fmax", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_mul", op="fmul", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_add", op="fadd", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub", op="fsub", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub_flipped", op="fsub", from=from, to=to, flipped=true) }}
|
||||
|
||||
|
||||
.clear:
|
||||
{% for r in range(from, to + 1) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.leaky_relu:
|
||||
add x2, x0, #8
|
||||
ld1 {v4.s}[0], [ x2 ]
|
||||
dup v4.4s, v4.s[0]
|
||||
|
||||
// bsl cond/dst, then, else
|
||||
// fcmge dst, src, #0.0
|
||||
{% for r in range(from, to + 1) %}
|
||||
fmul v0.4s, v{{r}}.4s, v4.4s
|
||||
fcmge v1.4s, v{{r}}.4s, #0.0
|
||||
bsl v1.16b, v{{r}}.16b, v0.16b
|
||||
and v{{r}}.16b, v1.16b, v1.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale:
|
||||
.q_shl:
|
||||
.q_shr:
|
||||
b .unsupported
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_i32_64x1_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_64x1_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .packed_packed_loop_1_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
ld1 {v9.s}[0], [ x2 ], 4
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x1 ], #64
|
||||
{% for reg in range(0, 4) %}
|
||||
mla v{{ reg * 2 + 16 }}.4s, v{{ reg * 2 }}.4s, v9.s[0]
|
||||
mla v{{ reg * 2 + 17 }}.4s, v{{ reg * 2 + 1 }}.4s, v9.s[0]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x1 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x1 ], #64
|
||||
{% for reg in range(0, 4) %}
|
||||
mla v{{ reg * 2 + 24 }}.4s, v{{ reg * 2 }}.4s, v9.s[0]
|
||||
mla v{{ reg * 2 + 25 }}.4s, v{{ reg * 2 + 1 }}.4s, v9.s[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1_i8i8:
|
||||
ld1 {v9.b}[0], [ x2 ], 1
|
||||
sshll v9.8h, v9.8b, 0
|
||||
|
||||
ld1 { v0.8b-v3.8b }, [ x1 ], #32
|
||||
ld1 { v4.8b-v7.8b }, [ x1 ], #32
|
||||
|
||||
{% for reg in range(0, 8) %}
|
||||
sshll v10.8h, v{{reg}}.8b, 0
|
||||
smlal v{{ reg * 2 + 16 }}.4s, v10.4h, v9.h[0]
|
||||
smlal2 v{{ reg * 2 + 17 }}.4s, v10.8h, v9.h[0]
|
||||
{% endfor %}
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1_i8i8
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
add v{{reg}}.4s, v{{reg}}.4s, v0.4s
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v15.s }[0], [ x3 ]
|
||||
xtn v15.4h, v15.4s
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x2 ], #64
|
||||
|
||||
{% for reg in range(0, 8) %}
|
||||
xtn v{{reg}}.4h, v{{reg}}.4s
|
||||
smlal v{{ reg + 16 }}.4s, v{{reg}}.4h, v15.h[0]
|
||||
{% endfor %}
|
||||
|
||||
ld1 { v0.4s-v3.4s }, [ x2 ], #64
|
||||
ld1 { v4.4s-v7.4s }, [ x2 ], #64
|
||||
|
||||
{% for reg in range(0, 8) %}
|
||||
xtn v{{reg}}.4h, v{{reg}}.4s
|
||||
smlal v{{ reg + 24 }}.4s, v{{reg}}.4h, v15.h[0]
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 64 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_cols.j2" %}
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.b }[{{ lane * 4 }}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for reg in range(16, 32) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{reg}}.s }[{{lane}}], [ x5 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd
|
||||
.global {{G}}arm64simd_mmm_i32_8x8_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_8x8_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .packed_packed_loop_1_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
mla v16.4s, v0.4s, v4.s[0]
|
||||
mla v17.4s, v1.4s, v4.s[0]
|
||||
mla v18.4s, v0.4s, v4.s[1]
|
||||
mla v19.4s, v1.4s, v4.s[1]
|
||||
|
||||
mla v20.4s, v0.4s, v4.s[2]
|
||||
mla v21.4s, v1.4s, v4.s[2]
|
||||
mla v22.4s, v0.4s, v4.s[3]
|
||||
mla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
mla v24.4s, v0.4s, v5.s[0]
|
||||
mla v25.4s, v1.4s, v5.s[0]
|
||||
mla v26.4s, v0.4s, v5.s[1]
|
||||
mla v27.4s, v1.4s, v5.s[1]
|
||||
|
||||
mla v28.4s, v0.4s, v5.s[2]
|
||||
mla v29.4s, v1.4s, v5.s[2]
|
||||
mla v30.4s, v0.4s, v5.s[3]
|
||||
mla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1_i8i8:
|
||||
|
||||
ld1 { v0.8b }, [ x1 ], #8
|
||||
sshll v0.8h, v0.8b, 0
|
||||
ld1 { v4.8b }, [ x2 ], #8
|
||||
sshll v4.8h, v4.8b, 0
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal2 v17.4s, v0.8h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal2 v19.4s, v0.8h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal2 v21.4s, v0.8h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal2 v23.4s, v0.8h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v4.h[4]
|
||||
smlal2 v25.4s, v0.8h, v4.h[4]
|
||||
smlal v26.4s, v0.4h, v4.h[5]
|
||||
smlal2 v27.4s, v0.8h, v4.h[5]
|
||||
smlal v28.4s, v0.4h, v4.h[6]
|
||||
smlal2 v29.4s, v0.8h, v4.h[6]
|
||||
smlal v30.4s, v0.4h, v4.h[7]
|
||||
smlal2 v31.4s, v0.8h, v4.h[7]
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1_i8i8
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
xtn v0.4h, v0.4s
|
||||
xtn v1.4h, v1.4s
|
||||
xtn v4.4h, v4.4s
|
||||
xtn v5.4h, v5.4s
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal v17.4s, v1.4h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal v19.4s, v1.4h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal v21.4s, v1.4h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal v23.4s, v1.4h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v5.h[0]
|
||||
smlal v25.4s, v1.4h, v5.h[0]
|
||||
smlal v26.4s, v0.4h, v5.h[1]
|
||||
smlal v27.4s, v1.4h, v5.h[1]
|
||||
smlal v28.4s, v0.4h, v5.h[2]
|
||||
smlal v29.4s, v1.4h, v5.h[2]
|
||||
smlal v30.4s, v0.4h, v5.h[3]
|
||||
smlal v31.4s, v1.4h, v5.h[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.j2" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.b }[{{ lane * 4 }}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
// vim: ft=arm
|
||||
|
||||
// C tile regs:
|
||||
// - x19-x29 to preserve (but x19, x28, x29 not used)
|
||||
// - d8..d15 to preserve
|
||||
// - v16 to v31, no need to preserve
|
||||
//
|
||||
// v16[0] v18[0] v20[0] v22[0] v24[0] v26[0] v28[0] v30[0]
|
||||
// v16[1] v18[1]
|
||||
// v16[2] v18[2]
|
||||
// v16[3] v18[3]
|
||||
//
|
||||
// v17[0] v19[0] v21[0] v23[0] v25[0] v27[0] v29[0] v31[0]
|
||||
// v17[1] v19[1]
|
||||
// v17[2] v19[2]
|
||||
// v17[3] v19[3]
|
||||
|
||||
// no preservation either for v0-v7...
|
||||
// packed A buffering (2x8 values): alternating v0, v1 with v2, v3
|
||||
// packed B buffering (2x8 values): alternating v4, v5 with v6, v7
|
||||
|
||||
.text
|
||||
.align 4
|
||||
|
||||
.cpu generic+fp+simd+dotprod
|
||||
.global {{G}}arm64simd_mmm_i32_8x8_dot_{{suffix}}
|
||||
{{G}}arm64simd_mmm_i32_8x8_dot_{{suffix}}:
|
||||
|
||||
/*
|
||||
prfm pldl1keep, [x1]
|
||||
prfm pldl1keep, [x2]
|
||||
*/
|
||||
stp x20, x21, [sp, #-16]!
|
||||
stp x22, x23, [sp, #-16]!
|
||||
stp x24, x25, [sp, #-16]!
|
||||
stp x26, x27, [sp, #-16]!
|
||||
|
||||
stp d8, d9, [sp, #-16]!
|
||||
stp d10, d11, [sp, #-16]!
|
||||
stp d12, d13, [sp, #-16]!
|
||||
stp d14, d15, [sp, #-16]!
|
||||
|
||||
{% include "dispatcher.j2" %}
|
||||
|
||||
.add_mat_mul:
|
||||
ldp x2, x4, [x0, #24] // b, packing
|
||||
ldp x3, x1, [x0, #8] // k, a
|
||||
|
||||
cmp x3, #0
|
||||
beq .non_linear_loop
|
||||
|
||||
cmp x4, #1
|
||||
beq .packed_packed_loop_1_i8i8
|
||||
|
||||
.packed_packed_loop_1:
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x1 ], #32
|
||||
ld1 { v4.4s, v5.4s }, [ x2 ], #32
|
||||
|
||||
mla v16.4s, v0.4s, v4.s[0]
|
||||
mla v17.4s, v1.4s, v4.s[0]
|
||||
mla v18.4s, v0.4s, v4.s[1]
|
||||
mla v19.4s, v1.4s, v4.s[1]
|
||||
|
||||
mla v20.4s, v0.4s, v4.s[2]
|
||||
mla v21.4s, v1.4s, v4.s[2]
|
||||
mla v22.4s, v0.4s, v4.s[3]
|
||||
mla v23.4s, v1.4s, v4.s[3]
|
||||
|
||||
mla v24.4s, v0.4s, v5.s[0]
|
||||
mla v25.4s, v1.4s, v5.s[0]
|
||||
mla v26.4s, v0.4s, v5.s[1]
|
||||
mla v27.4s, v1.4s, v5.s[1]
|
||||
|
||||
mla v28.4s, v0.4s, v5.s[2]
|
||||
mla v29.4s, v1.4s, v5.s[2]
|
||||
mla v30.4s, v0.4s, v5.s[3]
|
||||
mla v31.4s, v1.4s, v5.s[3]
|
||||
|
||||
subs x3, x3, #1
|
||||
bne .packed_packed_loop_1
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.packed_packed_loop_1_i8i8:
|
||||
// PackedI8K4 (K=4-inner, r=8): per 4-K block, A is m0-3 (v0) / m4-7 (v1),
|
||||
// B is n0-3 (v4) / n4-7 (v5), each lane a 4xi8 group. SDOT by-element dots
|
||||
// a B column's 4 K against all 4 m rows of an A half. Same v16..v31 tile
|
||||
// layout as the SMLAL kernel: v[16 + n*2 + m_half] = C[m_half*4..][n].
|
||||
ld1 { v0.16b, v1.16b }, [ x1 ], #32
|
||||
ld1 { v4.16b, v5.16b }, [ x2 ], #32
|
||||
|
||||
sdot v16.4s, v0.16b, v4.4b[0]
|
||||
sdot v17.4s, v1.16b, v4.4b[0]
|
||||
sdot v18.4s, v0.16b, v4.4b[1]
|
||||
sdot v19.4s, v1.16b, v4.4b[1]
|
||||
sdot v20.4s, v0.16b, v4.4b[2]
|
||||
sdot v21.4s, v1.16b, v4.4b[2]
|
||||
sdot v22.4s, v0.16b, v4.4b[3]
|
||||
sdot v23.4s, v1.16b, v4.4b[3]
|
||||
|
||||
sdot v24.4s, v0.16b, v5.4b[0]
|
||||
sdot v25.4s, v1.16b, v5.4b[0]
|
||||
sdot v26.4s, v0.16b, v5.4b[1]
|
||||
sdot v27.4s, v1.16b, v5.4b[1]
|
||||
sdot v28.4s, v0.16b, v5.4b[2]
|
||||
sdot v29.4s, v1.16b, v5.4b[2]
|
||||
sdot v30.4s, v0.16b, v5.4b[3]
|
||||
sdot v31.4s, v1.16b, v5.4b[3]
|
||||
|
||||
subs x3, x3, #4
|
||||
bgt .packed_packed_loop_1_i8i8
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_scalars.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_rows.j2" %}
|
||||
{% set mr = 8 %}{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_i32_per_cols.j2" %}
|
||||
{% set from = 16 %}{% set to = 31 %}{% include "arm64simd_mmm_load_tile.j2" %}
|
||||
|
||||
.add_unicast:
|
||||
ldp x5, x6, [x0, #8]
|
||||
ldp x7, x8, [x0, #24]
|
||||
|
||||
cmp x8, #4
|
||||
beq non_linear_addc_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.b}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
sshll v0.8h, v0.8b, 0
|
||||
sshll v0.4s, v0.4h, 0
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
non_linear_addc_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
ld1 {v0.s}[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
add v{{ col * 2 + reg }}.4s, v{{ col * 2 + reg }}.4s, v0.4s
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.add_row_col_products:
|
||||
ldr x2, [x0, #8]
|
||||
ldr x3, [x0, #16]
|
||||
|
||||
ld1 { v0.4s, v1.4s }, [ x2 ]
|
||||
ld1 { v4.4s, v5.4s }, [ x3 ]
|
||||
|
||||
xtn v0.4h, v0.4s
|
||||
xtn v1.4h, v1.4s
|
||||
xtn v4.4h, v4.4s
|
||||
xtn v5.4h, v5.4s
|
||||
|
||||
smlal v16.4s, v0.4h, v4.h[0]
|
||||
smlal v17.4s, v1.4h, v4.h[0]
|
||||
smlal v18.4s, v0.4h, v4.h[1]
|
||||
smlal v19.4s, v1.4h, v4.h[1]
|
||||
smlal v20.4s, v0.4h, v4.h[2]
|
||||
smlal v21.4s, v1.4h, v4.h[2]
|
||||
smlal v22.4s, v0.4h, v4.h[3]
|
||||
smlal v23.4s, v1.4h, v4.h[3]
|
||||
|
||||
smlal v24.4s, v0.4h, v5.h[0]
|
||||
smlal v25.4s, v1.4h, v5.h[0]
|
||||
smlal v26.4s, v0.4h, v5.h[1]
|
||||
smlal v27.4s, v1.4h, v5.h[1]
|
||||
smlal v28.4s, v0.4h, v5.h[2]
|
||||
smlal v29.4s, v1.4h, v5.h[2]
|
||||
smlal v30.4s, v0.4h, v5.h[3]
|
||||
smlal v31.4s, v1.4h, v5.h[3]
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
{% include "arm64simd_mmm_i32_scale_q16_q31.j2" %}
|
||||
|
||||
.store:
|
||||
ldp x5, x6, [x0, #8] // c base ptr, rsc
|
||||
ldp x7, x8, [x0, #24] // csc, item_size
|
||||
|
||||
cmp x8, #4
|
||||
beq .store_strides_i32
|
||||
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.b }[{{ lane * 4 }}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.store_strides_i32:
|
||||
{% for col in range(8, 16) %}
|
||||
mov x4, x5
|
||||
{% for reg in range(0, 2) %}
|
||||
{% for lane in range(0, 4) %}
|
||||
st1 { v{{ col * 2 + reg }}.s }[{{lane}}], [ x4 ], x6
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
add x5, x5, x7
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.return:
|
||||
ldp d14, d15, [sp], #16
|
||||
ldp d12, d13, [sp], #16
|
||||
ldp d10, d11, [sp], #16
|
||||
ldp d8, d9, [sp], #16
|
||||
|
||||
ldp x26, x27, [sp], #16
|
||||
ldp x24, x25, [sp], #16
|
||||
ldp x22, x23, [sp], #16
|
||||
ldp x20, x21, [sp], #16
|
||||
|
||||
ret
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_col %}
|
||||
|
||||
{{ per_col(label="per_col_min", op="smin", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_max", op="smax", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_mul", op="mul", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_add", op="add", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub", op="sub", mr=mr, from=from, to=to) }}
|
||||
{{ per_col(label="per_col_sub_flipped", op="sub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import per_row %}
|
||||
|
||||
{{ per_row(label="per_row_min", op="smin", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_max", op="smax", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_mul", op="mul", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_add", op="add", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub", op="sub", mr=mr, from=from, to=to) }}
|
||||
{{ per_row(label="per_row_sub_flipped", op="sub", mr=mr, from=from, to=to, flipped=true) }}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
// vim: ft=arm
|
||||
|
||||
{% from "arm64simd_mmm_4s_ops.j2" import scalar %}
|
||||
|
||||
{{ scalar(label="scalar_min", op="smin", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_max", op="smax", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_mul", op="mul", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_add", op="add", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub", op="sub", from=from, to=to) }}
|
||||
{{ scalar(label="scalar_sub_flipped", op="sub", from=from, to=to, flipped=true) }}
|
||||
|
||||
.clear:
|
||||
{% for r in range(from, to + 1) %}
|
||||
eor v{{r}}.8b, v{{r}}.8b, v{{r}}.8b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.leaky_relu:
|
||||
add x2, x0, #8
|
||||
ld1 {v4.s}[0], [ x2 ]
|
||||
dup v4.4s, v4.s[0]
|
||||
|
||||
// bsl cond/dst, then, else
|
||||
// fcmge dst, src, #0.0
|
||||
{% for r in range(from, to + 1) %}
|
||||
mul v0.4s, v{{r}}.4s, v4.4s
|
||||
cmge v1.4s, v{{r}}.4s, #0
|
||||
bsl v1.16b, v{{r}}.16b, v0.16b
|
||||
and v{{r}}.16b, v1.16b, v1.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
|
||||
// vim: ft=arm
|
||||
|
||||
.q_scale:
|
||||
ldp x5, x6, [x0, #8] // x5: shift, x6: policy
|
||||
add x2, x0, #24
|
||||
ld1r { v2.4s }, [x2] // v2.4s <- multiplier
|
||||
|
||||
mov w3, #1
|
||||
ins v4.d[0], x3
|
||||
dup v4.2d, v4.d[0] // v4.2d <- 1
|
||||
|
||||
add x5, x5, #32 // add 32 to shift
|
||||
neg x5, x5 // broadcast shift
|
||||
ins v1.d[0], x5
|
||||
dup v1.2d, v1.d[0] // v1.2s <- -(shift + 32)
|
||||
|
||||
cmp x6, 1
|
||||
beq .q_scale_rounding_zero
|
||||
cmp x6, 2
|
||||
beq .q_scale_rounding_away
|
||||
cmp x6, 3
|
||||
beq .q_scale_rounding_minus_inf
|
||||
cmp x6, 4
|
||||
beq .q_scale_rounding_plus_inf
|
||||
cmp x6, 5
|
||||
beq .q_scale_rounding_even
|
||||
cmp x6, 6
|
||||
beq .q_scale_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_scale_rounding_zero:
|
||||
// rust: signum * ((abs + nudge2) >> shift
|
||||
// asm: signum * (2*abs - 1) >>r (shift + 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sub v8.2d, v8.2d, v4.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sub v9.2d, v9.2d, v4.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_away: // signum * (abs >> (shift-1) + 1 >> 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_minus_inf: // val >> shift
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sub v8.2d, v8.2d, v4.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sub v9.2d, v9.2d, v4.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_plus_inf: // (val >> shift-1)+1 >>1
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_even: // signum * ((abs >> shift-1) + (abs & 0x1) - 1 >> 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqshl v3.2d, v8.2d, v1.2d // abs >> shift - 1
|
||||
and v3.16b, v3.16b, v4.16b // abs & 0x1
|
||||
sub v3.2d, v3.2d, v4.2d //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
add v8.2d, v8.2d, v3.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sqshl v3.2d, v9.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
sub v3.2d, v3.2d, v4.2d //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
add v9.2d, v9.2d, v3.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_scale_rounding_odd: // signum * ((abs >> shift-1) - (abs & 0x1) >> 1)
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
sqdmull v8.2d, v{{q}}.2s, v2.2s
|
||||
sqdmull2 v9.2d, v{{q}}.4s, v2.4s //mul without shift and store results in v8 and v9
|
||||
|
||||
sqshl v3.2d, v8.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
sub v8.2d, v8.2d, v3.2d
|
||||
sqrshl v8.2d, v8.2d, v1.2d
|
||||
|
||||
sqshl v3.2d, v9.2d, v1.2d
|
||||
and v3.16b, v3.16b, v4.16b //nudge : -1 if we want to round down, 0 if up
|
||||
|
||||
sub v9.2d, v9.2d, v3.2d
|
||||
sqrshl v9.2d, v9.2d, v1.2d
|
||||
|
||||
uzp1 v{{q}}.4s, v8.4s, v9.4s //combine back
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shl:
|
||||
ldr x5, [x0, #8] // x5: shift
|
||||
ins v1.s[0], w5
|
||||
dup v1.4s, v1.s[0] // v1.4s <- shift
|
||||
|
||||
{% for q in range(16, 32) %}
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr:
|
||||
ldp x5, x6, [x0, #8] // x5: shift, x6: policy
|
||||
|
||||
mov w3, #1
|
||||
ins v4.s[0], w3
|
||||
dup v4.4s, v4.s[0] // v4.4d <- 1
|
||||
|
||||
neg w5, w5 // broadcast shift
|
||||
ins v1.s[0], w5
|
||||
dup v1.4s, v1.s[0] // v1.4s <- -shift
|
||||
|
||||
cmp x6, 1
|
||||
beq .q_shr_rounding_zero
|
||||
cmp x6, 2
|
||||
beq .q_shr_rounding_away
|
||||
cmp x6, 3
|
||||
beq .q_shr_rounding_minus_inf
|
||||
cmp x6, 4
|
||||
beq .q_shr_rounding_plus_inf
|
||||
cmp x6, 5
|
||||
beq .q_shr_rounding_even
|
||||
cmp x6, 6
|
||||
beq .q_shr_rounding_odd
|
||||
|
||||
b .unsupported
|
||||
|
||||
.q_shr_rounding_zero:
|
||||
// asm: signum * (abs >>r shift)
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
|
||||
sub v{{q}}.4s, v{{q}}.4s, v4.4s
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_away:
|
||||
{% for q in range(16, 32) %}
|
||||
cmlt v0.4s, v{{q}}.4s, #0
|
||||
abs v{{q}}.4s, v{{q}}.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
|
||||
neg v3.4s, v{{q}}.4s
|
||||
bit v{{q}}.16b, v3.16b, v0.16b
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_minus_inf:
|
||||
{% for q in range(16, 32) %}
|
||||
sqneg v{{q}}.4s, v{{q}}.4s
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
sqneg v{{q}}.4s, v{{q}}.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_plus_inf:
|
||||
{% for q in range(16, 32) %}
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_even:
|
||||
// sqrshl is round(+inf), sqshl trauncates
|
||||
// we look at parity of result by truncation: if it's odd, we have nothing more to do, we go towards +inf
|
||||
// if it's even, we need to nudge towards 0 by adding -1
|
||||
// => nudge = (x >>l shift) & 0x1 - 1 (>>l is sqshl)
|
||||
// => result is (x + nudge) >>r shift (with sqrshl)
|
||||
{% for q in range(16, 32) %}
|
||||
sqshl v3.4s, v{{q}}.4s, v1.4s // trunc
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
sub v3.4s, v3.4s, v4.4s
|
||||
add v{{q}}.4s, v{{q}}.4s, v3.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
|
||||
.q_shr_rounding_odd:
|
||||
// here: nudge is -((x >>l shift) & 0x1)
|
||||
{% for q in range(16, 32) %}
|
||||
sqshl v3.4s, v{{q}}.4s, v1.4s // trunc
|
||||
and v3.16b, v3.16b, v4.16b
|
||||
neg v3.4s, v3.4s
|
||||
add v{{q}}.4s, v{{q}}.4s, v3.4s
|
||||
|
||||
sqrshl v{{q}}.4s, v{{q}}.4s, v1.4s
|
||||
{% endfor %}
|
||||
b .non_linear_loop
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user