#include "/srv/home/gc-sdk/ipu-stack/device/static_runtime.S"
#define GELU_WITH_BIAS
#define REFERENCE_GELU_WITH_BIAS
#include "arch/gc_tile_defines.h"

	.text
	.allow_optimizations

	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL
	.set REFERENCE_GELU_ALPHA_F16X2, 0x3a623a62
	.set REFERENCE_GELU_BETA_F16X2, 0x29b929b9
	.set ONE_F16X2, 0x3c003c00
	.set HALF_F16X2, 0x38003800
	// Callable supervisor ABI: m2 output, m3 input, m4 physical element
	// count, and m10 return. The contiguous entry accepts whole half pairs.

	.macro REFERENCE_GELU_PAIR value
	// tanh GeLU: x/2 * (1 + tanh(sqrt(2/pi) * (x + 0.044715*x^3))).
	ld32 $a2, $mvertex_base, $m15, 10
	f16v2min $a3, \value, $a2
	ld32 $a2, $mvertex_base, $m15, 12
	f16v2max $a3, $a3, $a2
	f16v2mul $a2, $a3, $a3
	f16v2mul $a2, $a2, $a3
	f16v2mul $a2, $a4, $a2
	f16v2add $a2, $a3, $a2
	f16v2mul $a2, $a5, $a2
	f16v2tanh $a2, $a2
	f16v2add $a2, $a6, $a2
	f16v2mul $a2, $a7, $a2
	f16v2mul $a2, \value, $a2
	.endm

	// Two identical pairs per bound let the vector loop load both in one issue.
	.macro STORE_REFERENCE_GELU_BOUNDS
	ldconst $m0, 0x48004800
	st32 $m0, $m11, $m15, 10
	st32 $m0, $m11, $m15, 11
	ldconst $m0, 0xc800c800
	st32 $m0, $m11, $m15, 12
	st32 $m0, $m11, $m15, 13
	.endm

	.macro LOAD_REFERENCE_GELU_CONSTANTS
	ldconst $a4, REFERENCE_GELU_BETA_F16X2
	ldconst $a5, REFERENCE_GELU_ALPHA_F16X2
	ldconst $a6, ONE_F16X2
	ldconst $a7, HALF_F16X2
	// Bound the tanh polynomial, not the final x: |x|>=8 is saturated.
	// Bounds live in the worker arguments; only eight ARF registers are writable.
	.endm

	.macro REFERENCE_GELU_TWO_PAIRS offset
#ifdef REFERENCE_GELU_WITH_BIAS
	ld32 $a2, $m9, $m15, \offset
	ld32 $a3, $m9, $m15, (\offset + 1)
	f16v4add $a0:1, $a0:1, $a2:3
	ldconst $a4, REFERENCE_GELU_BETA_F16X2
	ldconst $a5, REFERENCE_GELU_ALPHA_F16X2
#endif
	ld64 $a2:3, $mvertex_base, $m15, 5
	f16v4min $a0:1, $a0:1, $a2:3
	ld64 $a2:3, $mvertex_base, $m15, 6
	f16v4max $a0:1, $a0:1, $a2:3
	f16v4mul $a2:3, $a0:1, $a0:1
	f16v4mul $a2:3, $a2:3, $a0:1
	f16v4mul $a2:3, $a4:BL, $a2:3
	f16v4add $a2:3, $a0:1, $a2:3
	f16v4mul $a2:3, $a5:BL, $a2:3
	{ ld32 $a0, $m3, $m15, \offset
	  f16v2tanh $a2, $a2 }
	{ ld32 $a1, $m3, $m15, (\offset + 1)
	  f16v2tanh $a3, $a3 }
#ifdef REFERENCE_GELU_WITH_BIAS
	// Polynomial constants are dead here; use their registers to reload x+b.
	ld32 $a4, $m9, $m15, \offset
	ld32 $a5, $m9, $m15, (\offset + 1)
	f16v4add $a0:1, $a0:1, $a4:5
#endif
	f16v4add $a2:3, $a6:BL, $a2:3
	f16v4mul $a2:3, $a7:BL, $a2:3
	f16v4mul $a2:3, $a0:1, $a2:3
	.endm

	.macro REFERENCE_GELU_EIGHT_PAIRS
	ld32 $a0, $m3, $m15, 0
	ld32 $a1, $m3, $m15, 1
	REFERENCE_GELU_TWO_PAIRS 0
	st32 $a2, $m2, $m15, 0
	st32 $a3, $m2, $m15, 1
	ld32 $a0, $m3, $m15, 2
	ld32 $a1, $m3, $m15, 3
	REFERENCE_GELU_TWO_PAIRS 2
	st32 $a2, $m2, $m15, 2
	st32 $a3, $m2, $m15, 3
	ld32 $a0, $m3, $m15, 4
	ld32 $a1, $m3, $m15, 5
	REFERENCE_GELU_TWO_PAIRS 4
	st32 $a2, $m2, $m15, 4
	st32 $a3, $m2, $m15, 5
	ld32 $a0, $m3, $m15, 6
	ld32 $a1, $m3, $m15, 7
	REFERENCE_GELU_TWO_PAIRS 6
	st32 $a2, $m2, $m15, 6
	st32 $a3, $m2, $m15, 7
	.endm


#ifndef REFERENCE_GELU_MACROS_ONLY
	.macro REFERENCE_GELU_ENTRY symbol, worker
	.section .text.\symbol,"ax",@progbits
	.globl \symbol
	.p2align 2
	.type \symbol,@function
\symbol:
	.supervisor
	add $m11, $m11, -64
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
#ifdef REFERENCE_GELU_WITH_BIAS
	// m4 bias, m5 rows, m6 row width; the ordinary entry takes m4 elements.
	st32 $m5, $m11, $m15, 2
	st32 $m4, $m11, $m15, 5
	st32 $m6, $m11, $m15, 6
#else
	st32 $m4, $m11, $m15, 2
#endif
	STORE_REFERENCE_GELU_BOUNDS
	setzi $m0, \worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	add $m11, $m11, 64
	br $m10
	.size \symbol, .-\symbol
	.endm

#ifdef REFERENCE_GELU_WITH_BIAS
	REFERENCE_GELU_ENTRY reference_bias_gelu_f16, .Lreference_gelu_contiguous_worker
#else
	REFERENCE_GELU_ENTRY reference_gelu_tanh_approx_f16, .Lreference_gelu_contiguous_worker
#endif

	.worker
	.p2align 3
.Lreference_gelu_contiguous_worker:
#ifdef REFERENCE_GELU_WITH_BIAS
	ld32 $m7, $mvertex_base, $m15, 2
	setzi $m1, 0
.Lreference_gelu_row:
#endif
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
#ifdef REFERENCE_GELU_WITH_BIAS
	ld32 $m8, $mvertex_base, $m15, 6
	ld32 $m9, $mvertex_base, $m15, 5
	add $m2, $m2, $m1
	add $m3, $m3, $m1
#else
	ld32 $m8, $mvertex_base, $m15, 2
#endif
	shr $m8, $m8, 1
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 3
	cmpult $m6, $m4, $m8
	brz $m6, .Lreference_gelu_contiguous_done
	LOAD_REFERENCE_GELU_CONSTANTS
	shl $m5, $m4, 2
	add $m2, $m2, $m5
	add $m3, $m3, $m5
#ifdef REFERENCE_GELU_WITH_BIAS
	add $m9, $m9, $m5
#endif
	setzi $m5, 8
	add $m6, $m5, -1
	and $m6, $m8, $m6
	brnz $m6, .Lreference_gelu_contiguous_loop
.Lreference_gelu_contiguous_aligned_loop:
	REFERENCE_GELU_EIGHT_PAIRS
	add $m2, $m2, 192
	add $m3, $m3, 192
#ifdef REFERENCE_GELU_WITH_BIAS
	add $m9, $m9, 192
#endif
	add $m4, $m4, 48
	cmpult $m6, $m4, $m8
	brnz $m6, .Lreference_gelu_contiguous_aligned_loop
	bri .Lreference_gelu_contiguous_done
.Lreference_gelu_contiguous_loop:
	sub $m6, $m8, $m4
	cmpult $m6, $m6, $m5
	brnz $m6, .Lreference_gelu_contiguous_tail
	REFERENCE_GELU_EIGHT_PAIRS
	add $m2, $m2, 192
	add $m3, $m3, 192
#ifdef REFERENCE_GELU_WITH_BIAS
	add $m9, $m9, 192
#endif
	add $m4, $m4, 48
	cmpult $m6, $m4, $m8
	brnz $m6, .Lreference_gelu_contiguous_loop
	bri .Lreference_gelu_contiguous_done
.Lreference_gelu_contiguous_tail:
	ld32 $a0, $m3, $m15, 0
#ifdef REFERENCE_GELU_WITH_BIAS
	LOAD_REFERENCE_GELU_CONSTANTS
	ld32 $a2, $m9, $m15, 0
	f16v2add $a0, $a0, $a2
	add $m9, $m9, 4
#endif
	REFERENCE_GELU_PAIR $a0
	st32 $a2, $m2, $m15, 0
	add $m2, $m2, 4
	add $m3, $m3, 4
	add $m4, $m4, 1
	cmpult $m6, $m4, $m8
	brnz $m6, .Lreference_gelu_contiguous_tail
.Lreference_gelu_contiguous_done:
#ifdef REFERENCE_GELU_WITH_BIAS
	shl $m6, $m8, 2
	add $m1, $m1, $m6
	sub $m7, $m7, 1
	brnz $m7, .Lreference_gelu_row
#endif
	exitz $m15

#endif

#include "arch/gc_tile_defines.h"

	.text
	.allow_optimizations

	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL
	.set GELU_ALPHA_F16X2, 0x3a623a62
	.set GELU_BETA_F16X2, 0x29b929b9
	.set ONE_F16X2, 0x3c003c00
	.set HALF_F16X2, 0x38003800
	// Callable supervisor ABI: m2 output, m3 input, m4 physical element
	// count, and m10 return. The contiguous entry accepts whole half pairs.

	.macro GELU_PAIR value
	// tanh GeLU: x/2 * (1 + tanh(sqrt(2/pi) * (x + 0.044715*x^3))).
	ld32 $a2, $mvertex_base, $m15, 10
	f16v2min $a3, \value, $a2
	ld32 $a2, $mvertex_base, $m15, 12
	f16v2max $a3, $a3, $a2
	f16v2mul $a2, $a3, $a3
	f16v2mul $a2, $a2, $a3
	f16v2mul $a2, $a4, $a2
	f16v2add $a2, $a3, $a2
	f16v2mul $a2, $a5, $a2
	f16v2tanh $a2, $a2
	f16v2add $a2, $a6, $a2
	f16v2mul $a2, $a7, $a2
	f16v2mul $a2, \value, $a2
	.endm

	// Two identical pairs per bound let the vector loop load both in one issue.
	.macro STORE_GELU_BOUNDS
	ldconst $m0, 0x48004800
	st32 $m0, $m11, $m15, 10
	st32 $m0, $m11, $m15, 11
	ldconst $m0, 0xc800c800
	st32 $m0, $m11, $m15, 12
	st32 $m0, $m11, $m15, 13
	.endm

	.macro LOAD_GELU_CONSTANTS
	ldconst $a4, GELU_BETA_F16X2
	ldconst $a5, GELU_ALPHA_F16X2
	ldconst $a6, ONE_F16X2
	ldconst $a7, HALF_F16X2
	// Bound the tanh polynomial, not the final x: |x|>=8 is saturated.
	// Bounds live in the worker arguments; only eight ARF registers are writable.
	.endm

	.macro GELU_TWO_PAIRS offset
#ifdef GELU_WITH_BIAS
	ld32 $a2, $m9, $m15, \offset
	ld32 $a3, $m9, $m15, (\offset + 1)
	// TAS holds the polynomial coefficients, freeing a4:5 for unclamped x+b.
	f16v4add $a4:5, $a0:1, $a2:3
#endif
	ld64 $a2:3, $mvertex_base, $m15, 5
#ifdef GELU_WITH_BIAS
	f16v4min $a0:1, $a4:5, $a2:3
#else
	f16v4min $a0:1, $a0:1, $a2:3
#endif
	ld64 $a2:3, $mvertex_base, $m15, 6
	f16v4max $a0:1, $a0:1, $a2:3
	f16v4mul $a2:3, $a0:1, $a0:1
	f16v4mul $a2:3, $a2:3, $a0:1
#ifdef GELU_WITH_BIAS
	// alpha*beta*x^3 + alpha*x, with one FP32 dot product then FP16 rounding.
	f16v4mix $a0:1, $a2:3, $a0:1
	f16v4gacc $a2:3
#else
	f16v4mul $a2:3, $a4:BL, $a2:3
	f16v4add $a2:3, $a0:1, $a2:3
	f16v4mul $a2:3, $a5:BL, $a2:3
#endif
#ifdef GELU_WITH_BIAS
	f16v2tanh $a2, $a2
	f16v2tanh $a3, $a3
#else
	{ ld32 $a0, $m3, $m15, \offset
	  f16v2tanh $a2, $a2 }
	{ ld32 $a1, $m3, $m15, (\offset + 1)
	  f16v2tanh $a3, $a3 }
#endif
	f16v4add $a2:3, $a6:BL, $a2:3
	f16v4mul $a2:3, $a7:BL, $a2:3
#ifdef GELU_WITH_BIAS
	f16v4mul $a2:3, $a4:5, $a2:3
#else
	f16v4mul $a2:3, $a0:1, $a2:3
#endif
	.endm

	.macro GELU_EIGHT_PAIRS
	ld32 $a0, $m3, $m15, 0
	ld32 $a1, $m3, $m15, 1
	GELU_TWO_PAIRS 0
	st32 $a2, $m2, $m15, 0
	st32 $a3, $m2, $m15, 1
	ld32 $a0, $m3, $m15, 2
	ld32 $a1, $m3, $m15, 3
	GELU_TWO_PAIRS 2
	st32 $a2, $m2, $m15, 2
	st32 $a3, $m2, $m15, 3
	ld32 $a0, $m3, $m15, 4
	ld32 $a1, $m3, $m15, 5
	GELU_TWO_PAIRS 4
	st32 $a2, $m2, $m15, 4
	st32 $a3, $m2, $m15, 5
	ld32 $a0, $m3, $m15, 6
	ld32 $a1, $m3, $m15, 7
	GELU_TWO_PAIRS 6
	st32 $a2, $m2, $m15, 6
	st32 $a3, $m2, $m15, 7
	.endm


#ifndef GELU_MACROS_ONLY
	.macro GELU_ENTRY symbol, worker
	.section .text.\symbol,"ax",@progbits
	.globl \symbol
	.p2align 2
	.type \symbol,@function
\symbol:
	.supervisor
	add $m11, $m11, -64
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
#ifdef GELU_WITH_BIAS
	// m4 bias, m5 rows, m6 row width; the ordinary entry takes m4 elements.
	st32 $m5, $m11, $m15, 2
	st32 $m4, $m11, $m15, 5
	st32 $m6, $m11, $m15, 6
#else
	st32 $m4, $m11, $m15, 2
#endif
	STORE_GELU_BOUNDS
	setzi $m0, \worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	add $m11, $m11, 64
	br $m10
	.size \symbol, .-\symbol
	.endm

#ifdef GELU_WITH_BIAS
	GELU_ENTRY bias_gelu_f16, .Lgelu_contiguous_worker
#else
	GELU_ENTRY gelu_tanh_approx_f16, .Lgelu_contiguous_worker
#endif

	.worker
	.p2align 3
.Lgelu_contiguous_worker:
#ifdef GELU_WITH_BIAS
	ldconst $a0, 0x3a622891 // half(alpha), half(alpha*beta)
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld32 $m7, $mvertex_base, $m15, 2
	setzi $m1, 0
.Lgelu_row:
#endif
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
#ifdef GELU_WITH_BIAS
	ld32 $m8, $mvertex_base, $m15, 6
	ld32 $m9, $mvertex_base, $m15, 5
	add $m2, $m2, $m1
	add $m3, $m3, $m1
#else
	ld32 $m8, $mvertex_base, $m15, 2
#endif
	shr $m8, $m8, 1
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 3
	cmpult $m6, $m4, $m8
	brz $m6, .Lgelu_contiguous_done
	LOAD_GELU_CONSTANTS
	shl $m5, $m4, 2
	add $m2, $m2, $m5
	add $m3, $m3, $m5
#ifdef GELU_WITH_BIAS
	add $m9, $m9, $m5
#endif
	setzi $m5, 8
	add $m6, $m5, -1
	and $m6, $m8, $m6
	brnz $m6, .Lgelu_contiguous_loop
.Lgelu_contiguous_aligned_loop:
	GELU_EIGHT_PAIRS
	add $m2, $m2, 192
	add $m3, $m3, 192
#ifdef GELU_WITH_BIAS
	add $m9, $m9, 192
#endif
	add $m4, $m4, 48
	cmpult $m6, $m4, $m8
	brnz $m6, .Lgelu_contiguous_aligned_loop
	bri .Lgelu_contiguous_done
.Lgelu_contiguous_loop:
	sub $m6, $m8, $m4
	cmpult $m6, $m6, $m5
	brnz $m6, .Lgelu_contiguous_tail
	GELU_EIGHT_PAIRS
	add $m2, $m2, 192
	add $m3, $m3, 192
#ifdef GELU_WITH_BIAS
	add $m9, $m9, 192
#endif
	add $m4, $m4, 48
	cmpult $m6, $m4, $m8
	brnz $m6, .Lgelu_contiguous_loop
	bri .Lgelu_contiguous_done
.Lgelu_contiguous_tail:
#ifdef GELU_WITH_BIAS
	// The vector path used a4:5 for x+b. Restore just these coefficients once;
	// the scalar loop preserves them, and a6:7 still hold one and one-half.
	ldconst $a4, GELU_BETA_F16X2
	ldconst $a5, GELU_ALPHA_F16X2
#endif
.Lgelu_pair_tail:
	ld32 $a0, $m3, $m15, 0
#ifdef GELU_WITH_BIAS
	ld32 $a2, $m9, $m15, 0
	f16v2add $a0, $a0, $a2
	add $m9, $m9, 4
#endif
	GELU_PAIR $a0
	st32 $a2, $m2, $m15, 0
	add $m2, $m2, 4
	add $m3, $m3, 4
	add $m4, $m4, 1
	cmpult $m6, $m4, $m8
	brnz $m6, .Lgelu_pair_tail
.Lgelu_contiguous_done:
#ifdef GELU_WITH_BIAS
	shl $m6, $m8, 2
	add $m1, $m1, $m6
	sub $m7, $m7, 1
	brnz $m7, .Lgelu_row
#endif
	exitz $m15

#endif
#include "arch/gc_tile_defines.h"

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL
	.set RECIPROCAL_3_SHL17, 43691
	// Callable supervisor ABI: m2 output, m3 initial partial, m4 packed remote
	// partials, m5 remote-partial count, m6 physical half-element count, and
	// m10 return. Each remote partial occupies one contiguous element-count row.
	.section .text.reference_reduce_sum_f16,"ax",@progbits
	.globl reference_reduce_sum_f16
	.p2align 2
	.type reference_reduce_sum_f16,@function
reference_reduce_sum_f16:
	.supervisor
	add $m11, $m11, -24
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	st32 $m6, $m11, $m15, 4
	setzi $m0, .Lreference_reference_reduce_sum_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	add $m11, $m11, 24
	br $m10
	.size reference_reduce_sum_f16, .-reference_reduce_sum_f16

	.worker
	.p2align 3
.Lreference_reference_reduce_sum_f16_worker:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	ld32 $m0, $mvertex_base, $m15, 3
	ld32 $m9, $mvertex_base, $m15, 4
	shr $m9, $m9, 3
	shl $m1, $m9, 1
	get $m5, $WSR
	and $m5, $m5, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m6, $m5, $m9
	brz $m6, .Lreference_reduce_sum_done
	shl $m10, $m5, 4
	add $m2, $m2, $m10
	add $m3, $m3, $m10
	add $m4, $m4, $m10
	add $m6, $m9, 5
	sub $m6, $m6, $m5
	setzi $m10, RECIPROCAL_3_SHL17
	mul $m6, $m6, $m10
	shr $m6, $m6, 18
	brnzdec $m6, .Lreference_reduce_sum_loop
	bri .Lreference_reduce_sum_done
.Lreference_reduce_sum_loop:
	ld64step $a0:1, $mzero, $m3+=, 1
	ld64step $a2:3, $mzero, $m3+=, 11
	mov $m7, $m4
	add $m10, $m4, 8
	// Preload one partial, then consume the old registers while fetching the
	// next. A zero repeat count skips the body for a single remote partial.
	sub $m8, $m0, 1
	ld64step $a4:5, $mzero, $m4+=, $m1
	ld64step $a6:7, $mzero, $m10+=, $m1
	.p2align 3
	{ rpt $m8, 1
	  fnop }
	{ ld64step $a4:5, $mzero, $m4+=, $m1
	  f16v4add $a0:1, $a0:1, $a4:5 }
	{ ld64step $a6:7, $mzero, $m10+=, $m1
	  f16v4add $a2:3, $a2:3, $a6:7 }
	{ add $m4, $m7, 96
	  f16v4add $a0:1, $a0:1, $a4:5 }
	{ st64step $a0:1, $mzero, $m2+=, 1
	  f16v4add $a2:3, $a2:3, $a6:7 }
	st64step $a2:3, $mzero, $m2+=, 11
	brnzdec $m6, .Lreference_reduce_sum_loop
.Lreference_reduce_sum_done:
	exitz $m15

#include "arch/gc_tile_defines.h"

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL
	.set RECIPROCAL_3_SHL17, 43691
	// Callable supervisor ABI: m2 output, m3 initial partial, m4 packed remote
	// partials, m5 remote-partial count, m6 physical half-element count, and
	// m10 return. Each remote partial occupies one contiguous element-count row.
	.section .text.reduce_sum_f16,"ax",@progbits
	.globl reduce_sum_f16
	.p2align 2
	.type reduce_sum_f16,@function
reduce_sum_f16:
	.supervisor
	add $m11, $m11, -24
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	st32 $m6, $m11, $m15, 4
	setzi $m0, .Lreduce_sum_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	add $m11, $m11, 24
	br $m10
	.size reduce_sum_f16, .-reduce_sum_f16

	.worker
	.p2align 3
.Lreduce_sum_f16_worker:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	ld32 $m0, $mvertex_base, $m15, 3
	ld32 $m9, $mvertex_base, $m15, 4
	shr $m9, $m9, 3
	shl $m1, $m9, 1
	get $m5, $WSR
	and $m5, $m5, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m6, $m5, $m9
	brz $m6, .Lreduce_sum_done
	shl $m10, $m5, 4
	add $m2, $m2, $m10
	add $m3, $m3, $m10
	add $m4, $m4, $m10
	add $m6, $m9, 5
	sub $m6, $m6, $m5
	setzi $m10, RECIPROCAL_3_SHL17
	mul $m6, $m6, $m10
	shr $m6, $m6, 18
	brnzdec $m6, .Lreduce_sum_loop
	bri .Lreduce_sum_done
.Lreduce_sum_loop:
	ld64step $a0:1, $mzero, $m3+=, 1
	ld64step $a2:3, $mzero, $m3+=, 11
	mov $m7, $m4
	add $m10, $m4, 8
	// Preload one partial, then consume the old registers while fetching the
	// next. A zero repeat count skips the body for a single remote partial.
	sub $m8, $m0, 1
	ld64step $a4:5, $mzero, $m4+=, $m1
	ld64step $a6:7, $mzero, $m10+=, $m1
	.p2align 3
	{ rpt $m8, 1
	  fnop }
	{ ld64step $a4:5, $mzero, $m4+=, $m1
	  f16v4add $a0:1, $a0:1, $a4:5 }
	{ ld64step $a6:7, $mzero, $m10+=, $m1
	  f16v4add $a2:3, $a2:3, $a6:7 }
	{ add $m4, $m7, 96
	  f16v4add $a0:1, $a0:1, $a4:5 }
	{ st64step $a0:1, $mzero, $m2+=, 1
	  f16v4add $a2:3, $a2:3, $a6:7 }
	st64step $a2:3, $mzero, $m2+=, 11
	brnzdec $m6, .Lreduce_sum_loop
.Lreduce_sum_done:
	exitz $m15

.section .text.kernel_test_benign_fp,"ax",@progbits
.supervisor
.p2align 2
.globl kernel_test_benign_fp
kernel_test_benign_fp:
get $m0, $FP_ICTL
ldconst $m1, 0xfffffff8
and $m0, $m0, $m1
put $FP_ICTL, $m0
br $m10

.section .text.kernel_test_strict_fp,"ax",@progbits
.supervisor
.p2align 2
.globl kernel_test_strict_fp
kernel_test_strict_fp:
get $m0, $FP_ICTL
setzi $m1, 7
or $m0, $m0, $m1
put $FP_ICTL, $m0
br $m10

.section .text.kernel_test_poison_acc,"ax",@progbits
.supervisor
.p2align 2
.globl kernel_test_poison_acc
kernel_test_poison_acc:
setzi $m0, .Lpoison_acc
runall $m0, $mzero, 0
sync TEXCH_SYNCZONE_LOCAL
br $m10
.worker
.Lpoison_acc:
setzi $a0, 0
setzi $a1, 0
ldconst $a2, 0x7fc07fc0
ldconst $a3, 0x7fc07fc0
f16v4istacc $a0:1, $a0:1, $a2:3, 0
exitz $mzero