From 0fb0441220173be870dd88fb2b7499562247176e Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 19 Aug 2026 14:16:28 +0200 Subject: [PATCH 1/2] wasm: gate TRSM call counters behind OPENBLAS_WASM_TRSM_PROFILE Keep production STRSM/DTRSM free of always-on increments; enable the counters only when counting invocations. Signed-off-by: Julien Jerphanion --- kernel/wasm/trsm_kernel_LN_wasm128.c | 4 ++++ kernel/wasm/trsm_kernel_LT_wasm128.c | 4 ++++ kernel/wasm/trsm_kernel_RN_wasm128.c | 4 ++++ kernel/wasm/trsm_kernel_RT_wasm128.c | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/kernel/wasm/trsm_kernel_LN_wasm128.c b/kernel/wasm/trsm_kernel_LN_wasm128.c index e8bc6bd8c4..6b54ddc9fb 100644 --- a/kernel/wasm/trsm_kernel_LN_wasm128.c +++ b/kernel/wasm/trsm_kernel_LN_wasm128.c @@ -40,6 +40,7 @@ static FLOAT dm1 = -1.; +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE static unsigned long long openblas_wasm128_strsm_ln_calls = 0; unsigned long long openblas_wasm128_get_strsm_ln_calls(void) { @@ -57,6 +58,7 @@ void openblas_wasm128_reset_dtrsm_ln_calls(void) { openblas_wasm128_dtrsm_ln_calls = 0; } #endif +#endif #ifdef CONJ #define GEMM_KERNEL GEMM_KERNEL_L @@ -257,10 +259,12 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, #endif FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE openblas_wasm128_strsm_ln_calls += 1; #else openblas_wasm128_dtrsm_ln_calls += 1; +#endif #endif BLASLONG i, j; diff --git a/kernel/wasm/trsm_kernel_LT_wasm128.c b/kernel/wasm/trsm_kernel_LT_wasm128.c index 1c4f0ce870..9ac0a9a1f9 100644 --- a/kernel/wasm/trsm_kernel_LT_wasm128.c +++ b/kernel/wasm/trsm_kernel_LT_wasm128.c @@ -38,6 +38,7 @@ #include "common.h" +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE static unsigned long long openblas_wasm128_strsm_lt_calls = 0; unsigned long long openblas_wasm128_get_strsm_lt_calls(void) { @@ -55,6 +56,7 @@ void openblas_wasm128_reset_dtrsm_lt_calls(void) { openblas_wasm128_dtrsm_lt_calls = 0; } #endif +#endif static FLOAT dm1 = -1.; @@ -247,10 +249,12 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, #endif FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE openblas_wasm128_strsm_lt_calls += 1; #else openblas_wasm128_dtrsm_lt_calls += 1; +#endif #endif FLOAT *aa, *cc; diff --git a/kernel/wasm/trsm_kernel_RN_wasm128.c b/kernel/wasm/trsm_kernel_RN_wasm128.c index be19281acd..77332391b8 100644 --- a/kernel/wasm/trsm_kernel_RN_wasm128.c +++ b/kernel/wasm/trsm_kernel_RN_wasm128.c @@ -38,6 +38,7 @@ #include "common.h" +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE static unsigned long long openblas_wasm128_strsm_rn_calls = 0; unsigned long long openblas_wasm128_get_strsm_rn_calls(void) { @@ -55,6 +56,7 @@ void openblas_wasm128_reset_dtrsm_rn_calls(void) { openblas_wasm128_dtrsm_rn_calls = 0; } #endif +#endif static FLOAT dm1 = -1.; @@ -247,10 +249,12 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, #endif FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE openblas_wasm128_strsm_rn_calls += 1; #else openblas_wasm128_dtrsm_rn_calls += 1; +#endif #endif FLOAT *aa, *cc; diff --git a/kernel/wasm/trsm_kernel_RT_wasm128.c b/kernel/wasm/trsm_kernel_RT_wasm128.c index ebfa1437ca..5e4ca4ca83 100644 --- a/kernel/wasm/trsm_kernel_RT_wasm128.c +++ b/kernel/wasm/trsm_kernel_RT_wasm128.c @@ -40,6 +40,7 @@ static FLOAT dm1 = -1.; +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE static unsigned long long openblas_wasm128_strsm_rt_calls = 0; unsigned long long openblas_wasm128_get_strsm_rt_calls(void) { @@ -57,6 +58,7 @@ void openblas_wasm128_reset_dtrsm_rt_calls(void) { openblas_wasm128_dtrsm_rt_calls = 0; } #endif +#endif #ifdef CONJ #define GEMM_KERNEL GEMM_KERNEL_R @@ -260,10 +262,12 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, #endif FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ +#ifdef OPENBLAS_WASM_TRSM_PROFILE #ifndef DOUBLE openblas_wasm128_strsm_rt_calls += 1; #else openblas_wasm128_dtrsm_rt_calls += 1; +#endif #endif BLASLONG i, j; From 2ed3ca39403232a823686ce2a3b75335a7971a72 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Wed, 19 Aug 2026 14:17:22 +0200 Subject: [PATCH 2/2] wasm: add a 4x4 SIMD128 GEMM microkernel for SGEMM/DGEMM Switch WASM128_GENERIC to unroll 4 with matching ncopy/tcopy and TRMM 4x4 so packing width stays consistent. Use relaxed SIMD madd in the inner loop when compiled with -mrelaxed-simd. Signed-off-by: Julien Jerphanion --- Changelog.txt | 3 + Makefile.wasm | 2 +- kernel/wasm/KERNEL | 10 + kernel/wasm/KERNEL.WASM128_GENERIC | 16 +- kernel/wasm/gemmkernel_4x4_wasm128.c | 396 +++++++++++++++++++++++++++ param.h | 37 ++- 6 files changed, 454 insertions(+), 10 deletions(-) create mode 100644 kernel/wasm/gemmkernel_4x4_wasm128.c diff --git a/Changelog.txt b/Changelog.txt index 708db6a042..baa7276a2e 100644 --- a/Changelog.txt +++ b/Changelog.txt @@ -158,6 +158,9 @@ x86_64: wasm: - added optimized kernels for STRSM and DTRSM + - added a 4x4 WASM SIMD128 GEMM microkernel for SGEMM/DGEMM with matching + 4-wide packing and TRMM, using relaxed SIMD madd in the inner loop + - gated leftover WASM TRSM call counters behind OPENBLAS_WASM_TRSM_PROFILE ==================================================================== Version 0.3.32 diff --git a/Makefile.wasm b/Makefile.wasm index 230110abb0..feca75ed4c 100644 --- a/Makefile.wasm +++ b/Makefile.wasm @@ -1 +1 @@ -CCOMMON_OPT += -msimd128 +CCOMMON_OPT += -msimd128 -mrelaxed-simd diff --git a/kernel/wasm/KERNEL b/kernel/wasm/KERNEL index b379fd4243..0c8dcfcc29 100644 --- a/kernel/wasm/KERNEL +++ b/kernel/wasm/KERNEL @@ -95,10 +95,20 @@ DGEMVTKERNEL = ../riscv64/gemv_t.c CGEMVTKERNEL = ../riscv64/zgemv_t.c ZGEMVTKERNEL = ../riscv64/zgemv_t.c +# KERNEL.$(TARGET_CORE) is included first; keep its TRMM width if set. +# Unroll-4 GEMM packing uses trmm_*copy_4, which is wrong with a 2x2 kernel. +ifndef STRMMKERNEL STRMMKERNEL = ../generic/trmmkernel_2x2.c +endif +ifndef DTRMMKERNEL DTRMMKERNEL = ../generic/trmmkernel_2x2.c +endif +ifndef CTRMMKERNEL CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +endif +ifndef ZTRMMKERNEL ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +endif ifndef SGEMMKERNEL SGEMMKERNEL = ../generic/gemmkernel_2x2.c diff --git a/kernel/wasm/KERNEL.WASM128_GENERIC b/kernel/wasm/KERNEL.WASM128_GENERIC index 6eb59ddd52..0265085b1c 100644 --- a/kernel/wasm/KERNEL.WASM128_GENERIC +++ b/kernel/wasm/KERNEL.WASM128_GENERIC @@ -95,20 +95,20 @@ DGEMVTKERNEL = ../riscv64/gemv_t.c CGEMVTKERNEL = ../riscv64/zgemv_t.c ZGEMVTKERNEL = ../riscv64/zgemv_t.c -STRMMKERNEL = ../generic/trmmkernel_2x2.c -DTRMMKERNEL = ../generic/trmmkernel_2x2.c +STRMMKERNEL = ../generic/trmmkernel_4x4.c +DTRMMKERNEL = ../generic/trmmkernel_4x4.c CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c -SGEMMKERNEL = gemmkernel_wasm128.c -SGEMMONCOPY = ../generic/gemm_ncopy_2.c -SGEMMOTCOPY = ../generic/gemm_tcopy_2.c +SGEMMKERNEL = gemmkernel_4x4_wasm128.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) -DGEMMKERNEL = gemmkernel_wasm128.c -DGEMMONCOPY = ../generic/gemm_ncopy_2.c -DGEMMOTCOPY = ../generic/gemm_tcopy_2.c +DGEMMKERNEL = gemmkernel_4x4_wasm128.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) diff --git a/kernel/wasm/gemmkernel_4x4_wasm128.c b/kernel/wasm/gemmkernel_4x4_wasm128.c new file mode 100644 index 0000000000..4a4df38c34 --- /dev/null +++ b/kernel/wasm/gemmkernel_4x4_wasm128.c @@ -0,0 +1,396 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* + * WASM SIMD128 GEMM micro-kernel, 4x4 register tile. + * + * Packed-data contract matches generic gemmkernel_4x4.c / tcopy_4 / ncopy_4: + * A panel is [A(r0,k)..A(r3,k)] per k; B panel is [B(k,c0)..B(k,c3)] per k. + */ + +#include "common.h" +#include "../generic/conversion_macros.h" + +#if defined(__wasm_simd128__) +#include +#if defined(__wasm_relaxed_simd__) +#define MADD_F32(a, b, c) wasm_f32x4_relaxed_madd((a), (b), (c)) +#define MADD_F64(a, b, c) wasm_f64x2_relaxed_madd((a), (b), (c)) +#else +#define MADD_F32(a, b, c) wasm_f32x4_add((c), wasm_f32x4_mul((a), (b))) +#define MADD_F64(a, b, c) wasm_f64x2_add((c), wasm_f64x2_mul((a), (b))) +#endif +#endif + +#ifdef BGEMM +#define C_TO_F32 TO_F32 +#else +#define C_TO_F32 +#endif + +int CNAME(BLASLONG bm, BLASLONG bn, BLASLONG bk, FLOAT alpha, IFLOAT *ba, + IFLOAT *bb, FLOAT *C, BLASLONG ldc +#ifdef TRMMKERNEL + , + BLASLONG offset +#endif +) { + BLASLONG i, j, k; + FLOAT *C0, *C1, *C2, *C3; + IFLOAT *ptrba, *ptrbb; + FLOAT r0c0, r1c0, r2c0, r3c0; + FLOAT r0c1, r1c1, r2c1, r3c1; + FLOAT r0c2, r1c2, r2c2, r3c2; + FLOAT r0c3, r1c3, r2c3, r3c3; + IFLOAT a0, a1, a2, a3, b0, b1, b2, b3; + + for (j = 0; j < bn / 4; j += 1) { + C0 = C; + C1 = C0 + ldc; + C2 = C1 + ldc; + C3 = C2 + ldc; + ptrba = ba; + + for (i = 0; i < bm / 4; i += 1) { + ptrbb = bb; +#if defined(__wasm_simd128__) && !defined(BGEMM) +#ifndef DOUBLE + { + v128_t acc0 = wasm_f32x4_splat(0.0f); + v128_t acc1 = wasm_f32x4_splat(0.0f); + v128_t acc2 = wasm_f32x4_splat(0.0f); + v128_t acc3 = wasm_f32x4_splat(0.0f); + for (k = 0; k < bk; k += 1) { + v128_t va = wasm_v128_load(ptrba); + v128_t vb = wasm_v128_load(ptrbb); + v128_t vb0 = wasm_i32x4_shuffle(vb, vb, 0, 0, 0, 0); + v128_t vb1 = wasm_i32x4_shuffle(vb, vb, 1, 1, 1, 1); + v128_t vb2 = wasm_i32x4_shuffle(vb, vb, 2, 2, 2, 2); + v128_t vb3 = wasm_i32x4_shuffle(vb, vb, 3, 3, 3, 3); + acc0 = MADD_F32(va, vb0, acc0); + acc1 = MADD_F32(va, vb1, acc1); + acc2 = MADD_F32(va, vb2, acc2); + acc3 = MADD_F32(va, vb3, acc3); + ptrba += 4; + ptrbb += 4; + } + v128_t valpha = wasm_f32x4_splat(alpha); + wasm_v128_store(C0, MADD_F32(acc0, valpha, wasm_v128_load(C0))); + wasm_v128_store(C1, MADD_F32(acc1, valpha, wasm_v128_load(C1))); + wasm_v128_store(C2, MADD_F32(acc2, valpha, wasm_v128_load(C2))); + wasm_v128_store(C3, MADD_F32(acc3, valpha, wasm_v128_load(C3))); + } +#else + { + v128_t a0l = wasm_f64x2_splat(0.0), a0h = wasm_f64x2_splat(0.0); + v128_t a1l = wasm_f64x2_splat(0.0), a1h = wasm_f64x2_splat(0.0); + v128_t a2l = wasm_f64x2_splat(0.0), a2h = wasm_f64x2_splat(0.0); + v128_t a3l = wasm_f64x2_splat(0.0), a3h = wasm_f64x2_splat(0.0); + for (k = 0; k < bk; k += 1) { + v128_t va01 = wasm_v128_load(ptrba); + v128_t va23 = wasm_v128_load(ptrba + 2); + v128_t vb01 = wasm_v128_load(ptrbb); + v128_t vb23 = wasm_v128_load(ptrbb + 2); + v128_t b0v = wasm_i64x2_shuffle(vb01, vb01, 0, 0); + v128_t b1v = wasm_i64x2_shuffle(vb01, vb01, 1, 1); + v128_t b2v = wasm_i64x2_shuffle(vb23, vb23, 0, 0); + v128_t b3v = wasm_i64x2_shuffle(vb23, vb23, 1, 1); + a0l = MADD_F64(va01, b0v, a0l); + a0h = MADD_F64(va23, b0v, a0h); + a1l = MADD_F64(va01, b1v, a1l); + a1h = MADD_F64(va23, b1v, a1h); + a2l = MADD_F64(va01, b2v, a2l); + a2h = MADD_F64(va23, b2v, a2h); + a3l = MADD_F64(va01, b3v, a3l); + a3h = MADD_F64(va23, b3v, a3h); + ptrba += 4; + ptrbb += 4; + } + v128_t valpha = wasm_f64x2_splat(alpha); + wasm_v128_store(C0, MADD_F64(a0l, valpha, wasm_v128_load(C0))); + wasm_v128_store(C0 + 2, MADD_F64(a0h, valpha, wasm_v128_load(C0 + 2))); + wasm_v128_store(C1, MADD_F64(a1l, valpha, wasm_v128_load(C1))); + wasm_v128_store(C1 + 2, MADD_F64(a1h, valpha, wasm_v128_load(C1 + 2))); + wasm_v128_store(C2, MADD_F64(a2l, valpha, wasm_v128_load(C2))); + wasm_v128_store(C2 + 2, MADD_F64(a2h, valpha, wasm_v128_load(C2 + 2))); + wasm_v128_store(C3, MADD_F64(a3l, valpha, wasm_v128_load(C3))); + wasm_v128_store(C3 + 2, MADD_F64(a3h, valpha, wasm_v128_load(C3 + 2))); + } +#endif +#else + r0c0 = r1c0 = r2c0 = r3c0 = 0; + r0c1 = r1c1 = r2c1 = r3c1 = 0; + r0c2 = r1c2 = r2c2 = r3c2 = 0; + r0c3 = r1c3 = r2c3 = r3c3 = 0; + for (k = 0; k < bk; k += 1) { + b0 = ptrbb[0]; + b1 = ptrbb[1]; + b2 = ptrbb[2]; + b3 = ptrbb[3]; + a0 = ptrba[0]; + a1 = ptrba[1]; + a2 = ptrba[2]; + a3 = ptrba[3]; + r0c0 += TO_F32(a0) * TO_F32(b0); + r1c0 += TO_F32(a1) * TO_F32(b0); + r2c0 += TO_F32(a2) * TO_F32(b0); + r3c0 += TO_F32(a3) * TO_F32(b0); + r0c1 += TO_F32(a0) * TO_F32(b1); + r1c1 += TO_F32(a1) * TO_F32(b1); + r2c1 += TO_F32(a2) * TO_F32(b1); + r3c1 += TO_F32(a3) * TO_F32(b1); + r0c2 += TO_F32(a0) * TO_F32(b2); + r1c2 += TO_F32(a1) * TO_F32(b2); + r2c2 += TO_F32(a2) * TO_F32(b2); + r3c2 += TO_F32(a3) * TO_F32(b2); + r0c3 += TO_F32(a0) * TO_F32(b3); + r1c3 += TO_F32(a1) * TO_F32(b3); + r2c3 += TO_F32(a2) * TO_F32(b3); + r3c3 += TO_F32(a3) * TO_F32(b3); + ptrba += 4; + ptrbb += 4; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C0[1] = TO_OUTPUT(C_TO_F32(C0[1]) + r1c0 * ALPHA); + C0[2] = TO_OUTPUT(C_TO_F32(C0[2]) + r2c0 * ALPHA); + C0[3] = TO_OUTPUT(C_TO_F32(C0[3]) + r3c0 * ALPHA); + C1[0] = TO_OUTPUT(C_TO_F32(C1[0]) + r0c1 * ALPHA); + C1[1] = TO_OUTPUT(C_TO_F32(C1[1]) + r1c1 * ALPHA); + C1[2] = TO_OUTPUT(C_TO_F32(C1[2]) + r2c1 * ALPHA); + C1[3] = TO_OUTPUT(C_TO_F32(C1[3]) + r3c1 * ALPHA); + C2[0] = TO_OUTPUT(C_TO_F32(C2[0]) + r0c2 * ALPHA); + C2[1] = TO_OUTPUT(C_TO_F32(C2[1]) + r1c2 * ALPHA); + C2[2] = TO_OUTPUT(C_TO_F32(C2[2]) + r2c2 * ALPHA); + C2[3] = TO_OUTPUT(C_TO_F32(C2[3]) + r3c2 * ALPHA); + C3[0] = TO_OUTPUT(C_TO_F32(C3[0]) + r0c3 * ALPHA); + C3[1] = TO_OUTPUT(C_TO_F32(C3[1]) + r1c3 * ALPHA); + C3[2] = TO_OUTPUT(C_TO_F32(C3[2]) + r2c3 * ALPHA); + C3[3] = TO_OUTPUT(C_TO_F32(C3[3]) + r3c3 * ALPHA); +#endif + C0 += 4; + C1 += 4; + C2 += 4; + C3 += 4; + } + + if (bm & 2) { + ptrbb = bb; + r0c0 = r1c0 = 0; + r0c1 = r1c1 = 0; + r0c2 = r1c2 = 0; + r0c3 = r1c3 = 0; + for (k = 0; k < bk; k += 1) { + b0 = ptrbb[0]; + b1 = ptrbb[1]; + b2 = ptrbb[2]; + b3 = ptrbb[3]; + a0 = ptrba[0]; + a1 = ptrba[1]; + r0c0 += TO_F32(a0) * TO_F32(b0); + r1c0 += TO_F32(a1) * TO_F32(b0); + r0c1 += TO_F32(a0) * TO_F32(b1); + r1c1 += TO_F32(a1) * TO_F32(b1); + r0c2 += TO_F32(a0) * TO_F32(b2); + r1c2 += TO_F32(a1) * TO_F32(b2); + r0c3 += TO_F32(a0) * TO_F32(b3); + r1c3 += TO_F32(a1) * TO_F32(b3); + ptrba += 2; + ptrbb += 4; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C0[1] = TO_OUTPUT(C_TO_F32(C0[1]) + r1c0 * ALPHA); + C1[0] = TO_OUTPUT(C_TO_F32(C1[0]) + r0c1 * ALPHA); + C1[1] = TO_OUTPUT(C_TO_F32(C1[1]) + r1c1 * ALPHA); + C2[0] = TO_OUTPUT(C_TO_F32(C2[0]) + r0c2 * ALPHA); + C2[1] = TO_OUTPUT(C_TO_F32(C2[1]) + r1c2 * ALPHA); + C3[0] = TO_OUTPUT(C_TO_F32(C3[0]) + r0c3 * ALPHA); + C3[1] = TO_OUTPUT(C_TO_F32(C3[1]) + r1c3 * ALPHA); + C0 += 2; + C1 += 2; + C2 += 2; + C3 += 2; + } + if (bm & 1) { + ptrbb = bb; + r0c0 = r0c1 = r0c2 = r0c3 = 0; + for (k = 0; k < bk; k += 1) { + a0 = ptrba[0]; + r0c0 += TO_F32(a0) * TO_F32(ptrbb[0]); + r0c1 += TO_F32(a0) * TO_F32(ptrbb[1]); + r0c2 += TO_F32(a0) * TO_F32(ptrbb[2]); + r0c3 += TO_F32(a0) * TO_F32(ptrbb[3]); + ptrba += 1; + ptrbb += 4; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C1[0] = TO_OUTPUT(C_TO_F32(C1[0]) + r0c1 * ALPHA); + C2[0] = TO_OUTPUT(C_TO_F32(C2[0]) + r0c2 * ALPHA); + C3[0] = TO_OUTPUT(C_TO_F32(C3[0]) + r0c3 * ALPHA); + C0 += 1; + C1 += 1; + C2 += 1; + C3 += 1; + } + bb = bb + bk * 4; + C = C + ldc * 4; + } + + if (bn & 2) { + C0 = C; + C1 = C0 + ldc; + ptrba = ba; + for (i = 0; i < bm / 4; i += 1) { + ptrbb = bb; + r0c0 = r1c0 = r2c0 = r3c0 = 0; + r0c1 = r1c1 = r2c1 = r3c1 = 0; + for (k = 0; k < bk; k += 1) { + b0 = ptrbb[0]; + b1 = ptrbb[1]; + a0 = ptrba[0]; + a1 = ptrba[1]; + a2 = ptrba[2]; + a3 = ptrba[3]; + r0c0 += TO_F32(a0) * TO_F32(b0); + r1c0 += TO_F32(a1) * TO_F32(b0); + r2c0 += TO_F32(a2) * TO_F32(b0); + r3c0 += TO_F32(a3) * TO_F32(b0); + r0c1 += TO_F32(a0) * TO_F32(b1); + r1c1 += TO_F32(a1) * TO_F32(b1); + r2c1 += TO_F32(a2) * TO_F32(b1); + r3c1 += TO_F32(a3) * TO_F32(b1); + ptrba += 4; + ptrbb += 2; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C0[1] = TO_OUTPUT(C_TO_F32(C0[1]) + r1c0 * ALPHA); + C0[2] = TO_OUTPUT(C_TO_F32(C0[2]) + r2c0 * ALPHA); + C0[3] = TO_OUTPUT(C_TO_F32(C0[3]) + r3c0 * ALPHA); + C1[0] = TO_OUTPUT(C_TO_F32(C1[0]) + r0c1 * ALPHA); + C1[1] = TO_OUTPUT(C_TO_F32(C1[1]) + r1c1 * ALPHA); + C1[2] = TO_OUTPUT(C_TO_F32(C1[2]) + r2c1 * ALPHA); + C1[3] = TO_OUTPUT(C_TO_F32(C1[3]) + r3c1 * ALPHA); + C0 += 4; + C1 += 4; + } + if (bm & 2) { + ptrbb = bb; + r0c0 = r1c0 = r0c1 = r1c1 = 0; + for (k = 0; k < bk; k += 1) { + b0 = ptrbb[0]; + b1 = ptrbb[1]; + a0 = ptrba[0]; + a1 = ptrba[1]; + r0c0 += TO_F32(a0) * TO_F32(b0); + r1c0 += TO_F32(a1) * TO_F32(b0); + r0c1 += TO_F32(a0) * TO_F32(b1); + r1c1 += TO_F32(a1) * TO_F32(b1); + ptrba += 2; + ptrbb += 2; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C0[1] = TO_OUTPUT(C_TO_F32(C0[1]) + r1c0 * ALPHA); + C1[0] = TO_OUTPUT(C_TO_F32(C1[0]) + r0c1 * ALPHA); + C1[1] = TO_OUTPUT(C_TO_F32(C1[1]) + r1c1 * ALPHA); + C0 += 2; + C1 += 2; + } + if (bm & 1) { + ptrbb = bb; + r0c0 = r0c1 = 0; + for (k = 0; k < bk; k += 1) { + a0 = ptrba[0]; + r0c0 += TO_F32(a0) * TO_F32(ptrbb[0]); + r0c1 += TO_F32(a0) * TO_F32(ptrbb[1]); + ptrba += 1; + ptrbb += 2; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C1[0] = TO_OUTPUT(C_TO_F32(C1[0]) + r0c1 * ALPHA); + C0 += 1; + C1 += 1; + } + bb = bb + bk * 2; + C = C + ldc * 2; + } + + if (bn & 1) { + C0 = C; + ptrba = ba; + for (i = 0; i < bm / 4; i += 1) { + ptrbb = bb; + r0c0 = r1c0 = r2c0 = r3c0 = 0; + for (k = 0; k < bk; k += 1) { + b0 = ptrbb[0]; + a0 = ptrba[0]; + a1 = ptrba[1]; + a2 = ptrba[2]; + a3 = ptrba[3]; + r0c0 += TO_F32(a0) * TO_F32(b0); + r1c0 += TO_F32(a1) * TO_F32(b0); + r2c0 += TO_F32(a2) * TO_F32(b0); + r3c0 += TO_F32(a3) * TO_F32(b0); + ptrba += 4; + ptrbb += 1; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C0[1] = TO_OUTPUT(C_TO_F32(C0[1]) + r1c0 * ALPHA); + C0[2] = TO_OUTPUT(C_TO_F32(C0[2]) + r2c0 * ALPHA); + C0[3] = TO_OUTPUT(C_TO_F32(C0[3]) + r3c0 * ALPHA); + C0 += 4; + } + if (bm & 2) { + ptrbb = bb; + r0c0 = r1c0 = 0; + for (k = 0; k < bk; k += 1) { + b0 = ptrbb[0]; + a0 = ptrba[0]; + a1 = ptrba[1]; + r0c0 += TO_F32(a0) * TO_F32(b0); + r1c0 += TO_F32(a1) * TO_F32(b0); + ptrba += 2; + ptrbb += 1; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C0[1] = TO_OUTPUT(C_TO_F32(C0[1]) + r1c0 * ALPHA); + C0 += 2; + } + if (bm & 1) { + ptrbb = bb; + r0c0 = 0; + for (k = 0; k < bk; k += 1) { + r0c0 += TO_F32(ptrba[0]) * TO_F32(ptrbb[0]); + ptrba += 1; + ptrbb += 1; + } + C0[0] = TO_OUTPUT(C_TO_F32(C0[0]) + r0c0 * ALPHA); + C0 += 1; + } + } + + return 0; +} diff --git a/param.h b/param.h index f6ed94737e..f51fd51edd 100644 --- a/param.h +++ b/param.h @@ -3062,7 +3062,42 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SYMV_P 16 #endif -#if defined(RISCV64_GENERIC) || defined(WASM128_GENERIC) +#if defined(WASM128_GENERIC) +#define GEMM_DEFAULT_OFFSET_A 0 +#define GEMM_DEFAULT_OFFSET_B 0 +#define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL + +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 + +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 + +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 + +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 + +#define SGEMM_DEFAULT_P 128 +#define DGEMM_DEFAULT_P 128 +#define CGEMM_DEFAULT_P 96 +#define ZGEMM_DEFAULT_P 64 + +#define SGEMM_DEFAULT_Q 240 +#define DGEMM_DEFAULT_Q 120 +#define CGEMM_DEFAULT_Q 120 +#define ZGEMM_DEFAULT_Q 120 + +#define SGEMM_DEFAULT_R 12288 +#define DGEMM_DEFAULT_R 8192 +#define CGEMM_DEFAULT_R 4096 +#define ZGEMM_DEFAULT_R 4096 + +#define SYMV_P 16 +#endif + +#if defined(RISCV64_GENERIC) #define GEMM_DEFAULT_OFFSET_A 0 #define GEMM_DEFAULT_OFFSET_B 0 #define GEMM_DEFAULT_ALIGN (BLASLONG)0x03fffUL