#include "/srv/home/gc-sdk/ipu-stack/device/static_runtime.S"
#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_1
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_1
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_1
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_1_half_panel_\@
add $m2, $m2, $m8
.Lcheck_1_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_1
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_1
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_1
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_1
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_1
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_1
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_1
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_1
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_1
	.endr
	f16v4gacc $a2:3
	QUAD_check_1
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_1
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_1_whole
	setzi $m0, .Lcheck_1_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_1_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_1_split_sum
	bri .Lcheck_1_run_last
.Lcheck_1_whole:
	setzi $m0, .Lcheck_1_worker
.Lcheck_1_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_1_worker:
	CONFIG_check_1
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_1_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_1_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_1_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_1_max_panel:
	MAXP_check_1
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_1_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_1_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_1_max_scalar
	sub $m7, $m7, 1
.Lcheck_1_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_1_max_pair
.Lcheck_1_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_1_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_1_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_1_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_1_exp_panel:
	EXPP_check_1
	add $m3, $m3, $m8
	NEXT_check_1
	brnzdec $m7, .Lcheck_1_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_1_exp_tail:
	BEGIN_check_1
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_1_exp_odd
	sub $m7, $m7, 1
.Lcheck_1_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_1_exp_pair
.Lcheck_1_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_1_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_1_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_1_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_1_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_1_zero_pair
.Lcheck_1_zero_panels_setup:
	END_check_1
	NEXT_check_1
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_1_sum_ready
	sub $m7, $m7, 1
.Lcheck_1_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_1
	brnzdec $m7, .Lcheck_1_zero_panel
.Lcheck_1_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_1_row
.Lcheck_1_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_1
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_1_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_1_parity_\@
.Lcheck_1_even_\@:
	setzi $m4, 0
.Lcheck_1_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_1
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_1_nonempty_\@
	setzi $m10, 0
.Lcheck_1_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_1_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_1_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_1_clamped_\@
	mov $m10, $m1
.Lcheck_1_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_1_split_max:
	INIT_check_1
.Lcheck_1_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_1_done
	ROW_check_1
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_1_split_max_tail
	sub $m7, $m7, 1
.Lcheck_1_split_max_panel:
	MAXP_check_1
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_1_split_max_panel
.Lcheck_1_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_1_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_1_split_max_odd
	sub $m7, $m7, 1
.Lcheck_1_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_1_split_max_pair
.Lcheck_1_split_max_odd:
	brz $m1, .Lcheck_1_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_1_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_1_split_max_row

	.p2align 3
.Lcheck_1_split_exp:
	CONFIG_check_1
	INIT_check_1
.Lcheck_1_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_1_done
	ROW_check_1
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_1_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_1_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_1_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_1_split_exp_panel:
	EXPP_check_1
	add $m3, $m3, $m8
	NEXT_check_1
	brnzdec $m7, .Lcheck_1_split_exp_panel
.Lcheck_1_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_1_split_padding
	BEGIN_check_1
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_1_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_1_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_1_split_exp_pair
.Lcheck_1_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_1_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_1_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_1_split_after_tail
	sub $m7, $m7, 1
.Lcheck_1_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_1_split_zero_pair
.Lcheck_1_split_after_tail:
	END_check_1
	NEXT_check_1
.Lcheck_1_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_1_split_padding_capacity
	mov $m1, $m7
.Lcheck_1_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_1_split_store_sum
	sub $m7, $m7, 1
.Lcheck_1_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_1
	brnzdec $m7, .Lcheck_1_split_zero_panel
.Lcheck_1_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_1_split_exp_row

	.p2align 3
.Lcheck_1_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_1_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_1_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_1_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_2
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_2
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_2
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_2_half_panel_\@
add $m2, $m2, $m8
.Lcheck_2_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_2
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_2
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_2
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_2
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_2
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_2
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_2
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_2
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_2
	.endr
	f16v4gacc $a2:3
	QUAD_check_2
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_2
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_2_whole
	setzi $m0, .Lcheck_2_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_2_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_2_split_sum
	bri .Lcheck_2_run_last
.Lcheck_2_whole:
	setzi $m0, .Lcheck_2_worker
.Lcheck_2_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_2_worker:
	CONFIG_check_2
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_2_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_2_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_2_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_2_max_panel:
	MAXP_check_2
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_2_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_2_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_2_max_scalar
	sub $m7, $m7, 1
.Lcheck_2_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_2_max_pair
.Lcheck_2_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_2_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_2_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_2_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_2_exp_panel:
	EXPP_check_2
	add $m3, $m3, $m8
	NEXT_check_2
	brnzdec $m7, .Lcheck_2_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_2_exp_tail:
	BEGIN_check_2
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_2_exp_odd
	sub $m7, $m7, 1
.Lcheck_2_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_2_exp_pair
.Lcheck_2_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_2_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_2_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_2_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_2_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_2_zero_pair
.Lcheck_2_zero_panels_setup:
	END_check_2
	NEXT_check_2
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_2_sum_ready
	sub $m7, $m7, 1
.Lcheck_2_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_2
	brnzdec $m7, .Lcheck_2_zero_panel
.Lcheck_2_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_2_row
.Lcheck_2_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_2
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_2_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_2_parity_\@
.Lcheck_2_even_\@:
	setzi $m4, 0
.Lcheck_2_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_2
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_2_nonempty_\@
	setzi $m10, 0
.Lcheck_2_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_2_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_2_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_2_clamped_\@
	mov $m10, $m1
.Lcheck_2_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_2_split_max:
	INIT_check_2
.Lcheck_2_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_2_done
	ROW_check_2
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_2_split_max_tail
	sub $m7, $m7, 1
.Lcheck_2_split_max_panel:
	MAXP_check_2
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_2_split_max_panel
.Lcheck_2_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_2_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_2_split_max_odd
	sub $m7, $m7, 1
.Lcheck_2_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_2_split_max_pair
.Lcheck_2_split_max_odd:
	brz $m1, .Lcheck_2_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_2_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_2_split_max_row

	.p2align 3
.Lcheck_2_split_exp:
	CONFIG_check_2
	INIT_check_2
.Lcheck_2_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_2_done
	ROW_check_2
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_2_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_2_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_2_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_2_split_exp_panel:
	EXPP_check_2
	add $m3, $m3, $m8
	NEXT_check_2
	brnzdec $m7, .Lcheck_2_split_exp_panel
.Lcheck_2_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_2_split_padding
	BEGIN_check_2
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_2_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_2_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_2_split_exp_pair
.Lcheck_2_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_2_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_2_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_2_split_after_tail
	sub $m7, $m7, 1
.Lcheck_2_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_2_split_zero_pair
.Lcheck_2_split_after_tail:
	END_check_2
	NEXT_check_2
.Lcheck_2_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_2_split_padding_capacity
	mov $m1, $m7
.Lcheck_2_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_2_split_store_sum
	sub $m7, $m7, 1
.Lcheck_2_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_2
	brnzdec $m7, .Lcheck_2_split_zero_panel
.Lcheck_2_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_2_split_exp_row

	.p2align 3
.Lcheck_2_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_2_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_2_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_2_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_15
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_15
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_15
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_15_half_panel_\@
add $m2, $m2, $m8
.Lcheck_15_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_15
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_15
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_15
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_15
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_15
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_15
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_15
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_15
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_15
	.endr
	f16v4gacc $a2:3
	QUAD_check_15
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_15
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_15_whole
	setzi $m0, .Lcheck_15_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_15_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_15_split_sum
	bri .Lcheck_15_run_last
.Lcheck_15_whole:
	setzi $m0, .Lcheck_15_worker
.Lcheck_15_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_15_worker:
	CONFIG_check_15
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_15_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_15_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_15_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_15_max_panel:
	MAXP_check_15
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_15_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_15_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_15_max_scalar
	sub $m7, $m7, 1
.Lcheck_15_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_15_max_pair
.Lcheck_15_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_15_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_15_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_15_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_15_exp_panel:
	EXPP_check_15
	add $m3, $m3, $m8
	NEXT_check_15
	brnzdec $m7, .Lcheck_15_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_15_exp_tail:
	BEGIN_check_15
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_15_exp_odd
	sub $m7, $m7, 1
.Lcheck_15_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_15_exp_pair
.Lcheck_15_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_15_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_15_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_15_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_15_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_15_zero_pair
.Lcheck_15_zero_panels_setup:
	END_check_15
	NEXT_check_15
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_15_sum_ready
	sub $m7, $m7, 1
.Lcheck_15_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_15
	brnzdec $m7, .Lcheck_15_zero_panel
.Lcheck_15_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_15_row
.Lcheck_15_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_15
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_15_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_15_parity_\@
.Lcheck_15_even_\@:
	setzi $m4, 0
.Lcheck_15_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_15
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_15_nonempty_\@
	setzi $m10, 0
.Lcheck_15_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_15_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_15_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_15_clamped_\@
	mov $m10, $m1
.Lcheck_15_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_15_split_max:
	INIT_check_15
.Lcheck_15_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_15_done
	ROW_check_15
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_15_split_max_tail
	sub $m7, $m7, 1
.Lcheck_15_split_max_panel:
	MAXP_check_15
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_15_split_max_panel
.Lcheck_15_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_15_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_15_split_max_odd
	sub $m7, $m7, 1
.Lcheck_15_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_15_split_max_pair
.Lcheck_15_split_max_odd:
	brz $m1, .Lcheck_15_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_15_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_15_split_max_row

	.p2align 3
.Lcheck_15_split_exp:
	CONFIG_check_15
	INIT_check_15
.Lcheck_15_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_15_done
	ROW_check_15
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_15_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_15_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_15_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_15_split_exp_panel:
	EXPP_check_15
	add $m3, $m3, $m8
	NEXT_check_15
	brnzdec $m7, .Lcheck_15_split_exp_panel
.Lcheck_15_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_15_split_padding
	BEGIN_check_15
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_15_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_15_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_15_split_exp_pair
.Lcheck_15_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_15_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_15_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_15_split_after_tail
	sub $m7, $m7, 1
.Lcheck_15_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_15_split_zero_pair
.Lcheck_15_split_after_tail:
	END_check_15
	NEXT_check_15
.Lcheck_15_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_15_split_padding_capacity
	mov $m1, $m7
.Lcheck_15_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_15_split_store_sum
	sub $m7, $m7, 1
.Lcheck_15_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_15
	brnzdec $m7, .Lcheck_15_split_zero_panel
.Lcheck_15_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_15_split_exp_row

	.p2align 3
.Lcheck_15_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_15_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_15_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_15_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_16
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_16
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_16
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_16_half_panel_\@
add $m2, $m2, $m8
.Lcheck_16_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_16
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_16
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_16
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_16
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_16
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_16
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_16
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_16
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_16
	.endr
	f16v4gacc $a2:3
	QUAD_check_16
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_16
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_16_whole
	setzi $m0, .Lcheck_16_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_16_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_16_split_sum
	bri .Lcheck_16_run_last
.Lcheck_16_whole:
	setzi $m0, .Lcheck_16_worker
.Lcheck_16_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_16_worker:
	CONFIG_check_16
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_16_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_16_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_16_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_16_max_panel:
	MAXP_check_16
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_16_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_16_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_16_max_scalar
	sub $m7, $m7, 1
.Lcheck_16_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_16_max_pair
.Lcheck_16_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_16_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_16_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_16_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_16_exp_panel:
	EXPP_check_16
	add $m3, $m3, $m8
	NEXT_check_16
	brnzdec $m7, .Lcheck_16_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_16_exp_tail:
	BEGIN_check_16
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_16_exp_odd
	sub $m7, $m7, 1
.Lcheck_16_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_16_exp_pair
.Lcheck_16_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_16_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_16_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_16_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_16_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_16_zero_pair
.Lcheck_16_zero_panels_setup:
	END_check_16
	NEXT_check_16
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_16_sum_ready
	sub $m7, $m7, 1
.Lcheck_16_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_16
	brnzdec $m7, .Lcheck_16_zero_panel
.Lcheck_16_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_16_row
.Lcheck_16_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_16
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_16_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_16_parity_\@
.Lcheck_16_even_\@:
	setzi $m4, 0
.Lcheck_16_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_16
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_16_nonempty_\@
	setzi $m10, 0
.Lcheck_16_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_16_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_16_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_16_clamped_\@
	mov $m10, $m1
.Lcheck_16_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_16_split_max:
	INIT_check_16
.Lcheck_16_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_16_done
	ROW_check_16
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_16_split_max_tail
	sub $m7, $m7, 1
.Lcheck_16_split_max_panel:
	MAXP_check_16
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_16_split_max_panel
.Lcheck_16_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_16_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_16_split_max_odd
	sub $m7, $m7, 1
.Lcheck_16_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_16_split_max_pair
.Lcheck_16_split_max_odd:
	brz $m1, .Lcheck_16_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_16_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_16_split_max_row

	.p2align 3
.Lcheck_16_split_exp:
	CONFIG_check_16
	INIT_check_16
.Lcheck_16_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_16_done
	ROW_check_16
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_16_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_16_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_16_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_16_split_exp_panel:
	EXPP_check_16
	add $m3, $m3, $m8
	NEXT_check_16
	brnzdec $m7, .Lcheck_16_split_exp_panel
.Lcheck_16_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_16_split_padding
	BEGIN_check_16
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_16_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_16_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_16_split_exp_pair
.Lcheck_16_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_16_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_16_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_16_split_after_tail
	sub $m7, $m7, 1
.Lcheck_16_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_16_split_zero_pair
.Lcheck_16_split_after_tail:
	END_check_16
	NEXT_check_16
.Lcheck_16_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_16_split_padding_capacity
	mov $m1, $m7
.Lcheck_16_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_16_split_store_sum
	sub $m7, $m7, 1
.Lcheck_16_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_16
	brnzdec $m7, .Lcheck_16_split_zero_panel
.Lcheck_16_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_16_split_exp_row

	.p2align 3
.Lcheck_16_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_16_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_16_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_16_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_17
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_17
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_17
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_17_half_panel_\@
add $m2, $m2, $m8
.Lcheck_17_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_17
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_17
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_17
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_17
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_17
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_17
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_17
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_17
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_17
	.endr
	f16v4gacc $a2:3
	QUAD_check_17
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_17
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_17_whole
	setzi $m0, .Lcheck_17_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_17_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_17_split_sum
	bri .Lcheck_17_run_last
.Lcheck_17_whole:
	setzi $m0, .Lcheck_17_worker
.Lcheck_17_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_17_worker:
	CONFIG_check_17
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_17_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_17_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_17_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_17_max_panel:
	MAXP_check_17
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_17_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_17_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_17_max_scalar
	sub $m7, $m7, 1
.Lcheck_17_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_17_max_pair
.Lcheck_17_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_17_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_17_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_17_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_17_exp_panel:
	EXPP_check_17
	add $m3, $m3, $m8
	NEXT_check_17
	brnzdec $m7, .Lcheck_17_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_17_exp_tail:
	BEGIN_check_17
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_17_exp_odd
	sub $m7, $m7, 1
.Lcheck_17_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_17_exp_pair
.Lcheck_17_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_17_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_17_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_17_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_17_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_17_zero_pair
.Lcheck_17_zero_panels_setup:
	END_check_17
	NEXT_check_17
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_17_sum_ready
	sub $m7, $m7, 1
.Lcheck_17_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_17
	brnzdec $m7, .Lcheck_17_zero_panel
.Lcheck_17_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_17_row
.Lcheck_17_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_17
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_17_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_17_parity_\@
.Lcheck_17_even_\@:
	setzi $m4, 0
.Lcheck_17_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_17
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_17_nonempty_\@
	setzi $m10, 0
.Lcheck_17_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_17_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_17_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_17_clamped_\@
	mov $m10, $m1
.Lcheck_17_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_17_split_max:
	INIT_check_17
.Lcheck_17_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_17_done
	ROW_check_17
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_17_split_max_tail
	sub $m7, $m7, 1
.Lcheck_17_split_max_panel:
	MAXP_check_17
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_17_split_max_panel
.Lcheck_17_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_17_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_17_split_max_odd
	sub $m7, $m7, 1
.Lcheck_17_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_17_split_max_pair
.Lcheck_17_split_max_odd:
	brz $m1, .Lcheck_17_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_17_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_17_split_max_row

	.p2align 3
.Lcheck_17_split_exp:
	CONFIG_check_17
	INIT_check_17
.Lcheck_17_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_17_done
	ROW_check_17
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_17_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_17_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_17_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_17_split_exp_panel:
	EXPP_check_17
	add $m3, $m3, $m8
	NEXT_check_17
	brnzdec $m7, .Lcheck_17_split_exp_panel
.Lcheck_17_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_17_split_padding
	BEGIN_check_17
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_17_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_17_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_17_split_exp_pair
.Lcheck_17_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_17_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_17_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_17_split_after_tail
	sub $m7, $m7, 1
.Lcheck_17_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_17_split_zero_pair
.Lcheck_17_split_after_tail:
	END_check_17
	NEXT_check_17
.Lcheck_17_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_17_split_padding_capacity
	mov $m1, $m7
.Lcheck_17_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_17_split_store_sum
	sub $m7, $m7, 1
.Lcheck_17_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_17
	brnzdec $m7, .Lcheck_17_split_zero_panel
.Lcheck_17_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_17_split_exp_row

	.p2align 3
.Lcheck_17_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_17_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_17_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_17_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_31
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_31
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_31
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_31_half_panel_\@
add $m2, $m2, $m8
.Lcheck_31_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_31
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_31
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_31
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_31
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_31
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_31
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_31
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_31
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_31
	.endr
	f16v4gacc $a2:3
	QUAD_check_31
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_31
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_31_whole
	setzi $m0, .Lcheck_31_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_31_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_31_split_sum
	bri .Lcheck_31_run_last
.Lcheck_31_whole:
	setzi $m0, .Lcheck_31_worker
.Lcheck_31_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_31_worker:
	CONFIG_check_31
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_31_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_31_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_31_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_31_max_panel:
	MAXP_check_31
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_31_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_31_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_31_max_scalar
	sub $m7, $m7, 1
.Lcheck_31_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_31_max_pair
.Lcheck_31_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_31_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_31_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_31_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_31_exp_panel:
	EXPP_check_31
	add $m3, $m3, $m8
	NEXT_check_31
	brnzdec $m7, .Lcheck_31_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_31_exp_tail:
	BEGIN_check_31
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_31_exp_odd
	sub $m7, $m7, 1
.Lcheck_31_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_31_exp_pair
.Lcheck_31_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_31_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_31_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_31_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_31_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_31_zero_pair
.Lcheck_31_zero_panels_setup:
	END_check_31
	NEXT_check_31
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_31_sum_ready
	sub $m7, $m7, 1
.Lcheck_31_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_31
	brnzdec $m7, .Lcheck_31_zero_panel
.Lcheck_31_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_31_row
.Lcheck_31_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_31
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_31_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_31_parity_\@
.Lcheck_31_even_\@:
	setzi $m4, 0
.Lcheck_31_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_31
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_31_nonempty_\@
	setzi $m10, 0
.Lcheck_31_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_31_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_31_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_31_clamped_\@
	mov $m10, $m1
.Lcheck_31_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_31_split_max:
	INIT_check_31
.Lcheck_31_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_31_done
	ROW_check_31
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_31_split_max_tail
	sub $m7, $m7, 1
.Lcheck_31_split_max_panel:
	MAXP_check_31
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_31_split_max_panel
.Lcheck_31_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_31_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_31_split_max_odd
	sub $m7, $m7, 1
.Lcheck_31_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_31_split_max_pair
.Lcheck_31_split_max_odd:
	brz $m1, .Lcheck_31_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_31_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_31_split_max_row

	.p2align 3
.Lcheck_31_split_exp:
	CONFIG_check_31
	INIT_check_31
.Lcheck_31_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_31_done
	ROW_check_31
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_31_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_31_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_31_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_31_split_exp_panel:
	EXPP_check_31
	add $m3, $m3, $m8
	NEXT_check_31
	brnzdec $m7, .Lcheck_31_split_exp_panel
.Lcheck_31_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_31_split_padding
	BEGIN_check_31
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_31_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_31_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_31_split_exp_pair
.Lcheck_31_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_31_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_31_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_31_split_after_tail
	sub $m7, $m7, 1
.Lcheck_31_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_31_split_zero_pair
.Lcheck_31_split_after_tail:
	END_check_31
	NEXT_check_31
.Lcheck_31_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_31_split_padding_capacity
	mov $m1, $m7
.Lcheck_31_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_31_split_store_sum
	sub $m7, $m7, 1
.Lcheck_31_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_31
	brnzdec $m7, .Lcheck_31_split_zero_panel
.Lcheck_31_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_31_split_exp_row

	.p2align 3
.Lcheck_31_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_31_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_31_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_31_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 64
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_64
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_64
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_64
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_64_half_panel_\@
add $m2, $m2, $m8
.Lcheck_64_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_64
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_64
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_64
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_64
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_64
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_64
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_64
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_64
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_64
	.endr
	f16v4gacc $a2:3
	QUAD_check_64
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_64
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_64_whole
	setzi $m0, .Lcheck_64_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_64_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_64_split_sum
	bri .Lcheck_64_run_last
.Lcheck_64_whole:
	setzi $m0, .Lcheck_64_worker
.Lcheck_64_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_64_worker:
	CONFIG_check_64
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_64_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_64_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_64_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_64_max_panel:
	MAXP_check_64
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_64_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_64_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_64_max_scalar
	sub $m7, $m7, 1
.Lcheck_64_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_64_max_pair
.Lcheck_64_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_64_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_64_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_64_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_64_exp_panel:
	EXPP_check_64
	add $m3, $m3, $m8
	NEXT_check_64
	brnzdec $m7, .Lcheck_64_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_64_exp_tail:
	BEGIN_check_64
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_64_exp_odd
	sub $m7, $m7, 1
.Lcheck_64_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_64_exp_pair
.Lcheck_64_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_64_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_64_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_64_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_64_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_64_zero_pair
.Lcheck_64_zero_panels_setup:
	END_check_64
	NEXT_check_64
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_64_sum_ready
	sub $m7, $m7, 1
.Lcheck_64_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_64
	brnzdec $m7, .Lcheck_64_zero_panel
.Lcheck_64_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_64_row
.Lcheck_64_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_64
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_64_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_64_parity_\@
.Lcheck_64_even_\@:
	setzi $m4, 0
.Lcheck_64_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_64
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_64_nonempty_\@
	setzi $m10, 0
.Lcheck_64_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_64_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_64_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_64_clamped_\@
	mov $m10, $m1
.Lcheck_64_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_64_split_max:
	INIT_check_64
.Lcheck_64_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_64_done
	ROW_check_64
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_64_split_max_tail
	sub $m7, $m7, 1
.Lcheck_64_split_max_panel:
	MAXP_check_64
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_64_split_max_panel
.Lcheck_64_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_64_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_64_split_max_odd
	sub $m7, $m7, 1
.Lcheck_64_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_64_split_max_pair
.Lcheck_64_split_max_odd:
	brz $m1, .Lcheck_64_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_64_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_64_split_max_row

	.p2align 3
.Lcheck_64_split_exp:
	CONFIG_check_64
	INIT_check_64
.Lcheck_64_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_64_done
	ROW_check_64
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_64_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_64_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_64_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_64_split_exp_panel:
	EXPP_check_64
	add $m3, $m3, $m8
	NEXT_check_64
	brnzdec $m7, .Lcheck_64_split_exp_panel
.Lcheck_64_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_64_split_padding
	BEGIN_check_64
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_64_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_64_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_64_split_exp_pair
.Lcheck_64_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_64_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_64_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_64_split_after_tail
	sub $m7, $m7, 1
.Lcheck_64_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_64_split_zero_pair
.Lcheck_64_split_after_tail:
	END_check_64
	NEXT_check_64
.Lcheck_64_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_64_split_padding_capacity
	mov $m1, $m7
.Lcheck_64_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_64_split_store_sum
	sub $m7, $m7, 1
.Lcheck_64_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_64
	brnzdec $m7, .Lcheck_64_split_zero_panel
.Lcheck_64_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_64_split_exp_row

	.p2align 3
.Lcheck_64_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_64_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_64_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_64_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 96
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_65
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_65
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_65
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_65_half_panel_\@
add $m2, $m2, $m8
.Lcheck_65_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_65
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_65
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_65
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_65
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_65
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_65
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_65
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_65
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_65
	.endr
	f16v4gacc $a2:3
	QUAD_check_65
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_65
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_65_whole
	setzi $m0, .Lcheck_65_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_65_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_65_split_sum
	bri .Lcheck_65_run_last
.Lcheck_65_whole:
	setzi $m0, .Lcheck_65_worker
.Lcheck_65_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_65_worker:
	CONFIG_check_65
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_65_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_65_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_65_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_65_max_panel:
	MAXP_check_65
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_65_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_65_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_65_max_scalar
	sub $m7, $m7, 1
.Lcheck_65_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_65_max_pair
.Lcheck_65_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_65_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_65_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_65_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_65_exp_panel:
	EXPP_check_65
	add $m3, $m3, $m8
	NEXT_check_65
	brnzdec $m7, .Lcheck_65_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_65_exp_tail:
	BEGIN_check_65
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_65_exp_odd
	sub $m7, $m7, 1
.Lcheck_65_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_65_exp_pair
.Lcheck_65_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_65_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_65_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_65_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_65_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_65_zero_pair
.Lcheck_65_zero_panels_setup:
	END_check_65
	NEXT_check_65
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_65_sum_ready
	sub $m7, $m7, 1
.Lcheck_65_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_65
	brnzdec $m7, .Lcheck_65_zero_panel
.Lcheck_65_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_65_row
.Lcheck_65_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_65
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_65_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_65_parity_\@
.Lcheck_65_even_\@:
	setzi $m4, 0
.Lcheck_65_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_65
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_65_nonempty_\@
	setzi $m10, 0
.Lcheck_65_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_65_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_65_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_65_clamped_\@
	mov $m10, $m1
.Lcheck_65_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_65_split_max:
	INIT_check_65
.Lcheck_65_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_65_done
	ROW_check_65
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_65_split_max_tail
	sub $m7, $m7, 1
.Lcheck_65_split_max_panel:
	MAXP_check_65
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_65_split_max_panel
.Lcheck_65_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_65_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_65_split_max_odd
	sub $m7, $m7, 1
.Lcheck_65_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_65_split_max_pair
.Lcheck_65_split_max_odd:
	brz $m1, .Lcheck_65_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_65_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_65_split_max_row

	.p2align 3
.Lcheck_65_split_exp:
	CONFIG_check_65
	INIT_check_65
.Lcheck_65_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_65_done
	ROW_check_65
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_65_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_65_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_65_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_65_split_exp_panel:
	EXPP_check_65
	add $m3, $m3, $m8
	NEXT_check_65
	brnzdec $m7, .Lcheck_65_split_exp_panel
.Lcheck_65_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_65_split_padding
	BEGIN_check_65
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_65_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_65_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_65_split_exp_pair
.Lcheck_65_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_65_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_65_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_65_split_after_tail
	sub $m7, $m7, 1
.Lcheck_65_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_65_split_zero_pair
.Lcheck_65_split_after_tail:
	END_check_65
	NEXT_check_65
.Lcheck_65_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_65_split_padding_capacity
	mov $m1, $m7
.Lcheck_65_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_65_split_store_sum
	sub $m7, $m7, 1
.Lcheck_65_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_65
	brnzdec $m7, .Lcheck_65_split_zero_panel
.Lcheck_65_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_65_split_exp_row

	.p2align 3
.Lcheck_65_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_65_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_65_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_65_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 768
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_729
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_729
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_729
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_729_half_panel_\@
add $m2, $m2, $m8
.Lcheck_729_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_729
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_729
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_729
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_729
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_729
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_729
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_729
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_729
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_729
	.endr
	f16v4gacc $a2:3
	QUAD_check_729
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_729
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_729_whole
	setzi $m0, .Lcheck_729_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_729_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_729_split_sum
	bri .Lcheck_729_run_last
.Lcheck_729_whole:
	setzi $m0, .Lcheck_729_worker
.Lcheck_729_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_729_worker:
	CONFIG_check_729
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_729_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_729_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_729_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_729_max_panel:
	MAXP_check_729
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_729_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_729_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_729_max_scalar
	sub $m7, $m7, 1
.Lcheck_729_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_729_max_pair
.Lcheck_729_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_729_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_729_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_729_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_729_exp_panel:
	EXPP_check_729
	add $m3, $m3, $m8
	NEXT_check_729
	brnzdec $m7, .Lcheck_729_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_729_exp_tail:
	BEGIN_check_729
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_729_exp_odd
	sub $m7, $m7, 1
.Lcheck_729_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_729
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_729_exp_pair
.Lcheck_729_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_729_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_729_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_729_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_729_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_729_zero_pair
.Lcheck_729_zero_panels_setup:
	END_check_729
	NEXT_check_729
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_729_sum_ready
	sub $m7, $m7, 1
.Lcheck_729_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_729
	brnzdec $m7, .Lcheck_729_zero_panel
.Lcheck_729_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_729_row
.Lcheck_729_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_729
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_729_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_729_parity_\@
.Lcheck_729_even_\@:
	setzi $m4, 0
.Lcheck_729_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_729
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_729_nonempty_\@
	setzi $m10, 0
.Lcheck_729_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_729_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_729_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_729_clamped_\@
	mov $m10, $m1
.Lcheck_729_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_729_split_max:
	INIT_check_729
.Lcheck_729_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_729_done
	ROW_check_729
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_729_split_max_tail
	sub $m7, $m7, 1
.Lcheck_729_split_max_panel:
	MAXP_check_729
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_729_split_max_panel
.Lcheck_729_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_729_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_729_split_max_odd
	sub $m7, $m7, 1
.Lcheck_729_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_729_split_max_pair
.Lcheck_729_split_max_odd:
	brz $m1, .Lcheck_729_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_729_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_729_split_max_row

	.p2align 3
.Lcheck_729_split_exp:
	CONFIG_check_729
	INIT_check_729
.Lcheck_729_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_729_done
	ROW_check_729
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_729_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_729_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_729_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_729_split_exp_panel:
	EXPP_check_729
	add $m3, $m3, $m8
	NEXT_check_729
	brnzdec $m7, .Lcheck_729_split_exp_panel
.Lcheck_729_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_729_split_padding
	BEGIN_check_729
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_729_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_729_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_729
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_729_split_exp_pair
.Lcheck_729_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_729_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_729_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_729_split_after_tail
	sub $m7, $m7, 1
.Lcheck_729_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_729_split_zero_pair
.Lcheck_729_split_after_tail:
	END_check_729
	NEXT_check_729
.Lcheck_729_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_729_split_padding_capacity
	mov $m1, $m7
.Lcheck_729_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_729_split_store_sum
	sub $m7, $m7, 1
.Lcheck_729_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_729
	brnzdec $m7, .Lcheck_729_split_zero_panel
.Lcheck_729_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_729_split_exp_row

	.p2align 3
.Lcheck_729_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_729_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_729_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_729_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 768
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_check_768
#undef SOFTMAX_FRAME_BYTES
#undef SOFTMAX_ROWS
#undef SOFTMAX_KEYS
#undef SOFTMAX_SPLIT
#undef ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_F8
#define ATTENTION_OUTPUT_SCALE -4
#include "arch/gc_tile_defines.h"
// FP8 probability stores; maxima/denominators remain FP32. The packed
// matrix has 32-key panels, while score reads still use 16-key FP16 panels.
#ifdef ATTENTION_OUTPUT_F8
#define SOFTMAX_ELEMENT_BYTES 1
#else
#define SOFTMAX_ELEMENT_BYTES 2
#endif

.macro CONFIG_check_768
#ifdef ATTENTION_OUTPUT_F8
setzi $a0, 2
put 9, $a0
setzi $a0, ((-ATTENTION_OUTPUT_SCALE) & 255)
put 10, $a0
#endif
.endm

.macro NEXT_check_768
#ifdef ATTENTION_OUTPUT_F8
ld32 $m0, $mvertex_base, $m15, 0
sub $m0, $m2, $m0
and $m0, $m0, 31
brnz $m0, .Lcheck_768_half_panel_\@
add $m2, $m2, $m8
.Lcheck_768_half_panel_\@:
#else
add $m2, $m2, $m8
#endif
.endm

.macro EIGHT_check_768
f16v2add $a6, $a6, $a0
f16v2add $a6, $a6, $a1
f16v2add $a6, $a6, $a2
f16v2add $a6, $a6, $a3
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endm

// Only a masked tail uses the explicit FP16 workspace (32 bytes per row).
// Complete panels never round-trip SRAM.
.macro BEGIN_check_768
#ifdef ATTENTION_OUTPUT_F8
mov $m1, $m2
ld32 $m2, $mvertex_base, $m15, 6
shl $m0, $m4, 5
add $m2, $m2, $m0
#endif
.endm

.macro END_check_768
#ifdef ATTENTION_OUTPUT_F8
add $m0, $m2, -32
mov $m2, $m1
.rept 2
ld64step $a0:1, $mzero, $m0+=, 1
ld64step $a2:3, $mzero, $m0+=, 1
f16v8tof8 $a0:1, $a0:3
st64step $a0:1, $mzero, $m2+=, 1
.endr
#endif
.endm
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_check_768
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_check_768
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_check_768
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// The caller clears accumulators once; previous panels leave finite values.
	// MIX then returns each preceding quad while starting the next one.
#ifdef ATTENTION_OUTPUT_F8
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	{ ld64step $a2:3, $mzero, $m3+=, 1; f16v2exp $a1, $a1 }
	f16v4mix $a2:3, $a2:3, $a4:5
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_768
	ld64step $a2:3, $mzero, $m3+=, 1
	f16v4mix $a0:1, $a2:3, $a4:5
	f16v2exp $a0, $a0
	f16v2exp $a1, $a1
	f16v4gacc $a2:3
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	EIGHT_check_768
#else
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_check_768
	.endr
	f16v4gacc $a2:3
	QUAD_check_768
#endif
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_check_768
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left probabilities, separate FP32 statistics/reduction workspace, and
// optional FP16 masked-tail workspace. ABI: m2 probabilities, m3 scores,
// m4 statistics, m5 reduction workspace, [m6 tail workspace], then rows/keys/mode.
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 28
#define SOFTMAX_ROWS $m7
#define SOFTMAX_KEYS $m8
#define SOFTMAX_SPLIT $m9
#else
#define SOFTMAX_FRAME_BYTES 24
#define SOFTMAX_ROWS $m6
#define SOFTMAX_KEYS $m7
#define SOFTMAX_SPLIT $m8
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 SOFTMAX_ROWS, $m11, $m15, 2
	st32 SOFTMAX_KEYS, $m11, $m15, 3
	st32 $m4, $m11, $m15, 4
	st32 $m5, $m11, $m15, 5
#if defined(ATTENTION_OUTPUT_F8) && !ATTENTION_FULL_BLOCK
	st32 $m6, $m11, $m15, 6
#endif
	brz SOFTMAX_SPLIT, .Lcheck_768_whole
	setzi $m0, .Lcheck_768_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_768_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lcheck_768_split_sum
	bri .Lcheck_768_run_last
.Lcheck_768_whole:
	setzi $m0, .Lcheck_768_worker
.Lcheck_768_run_last:
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lcheck_768_worker:
	CONFIG_check_768
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_768_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lcheck_768_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_768_max_tail
	sub $m7, $m7, 1
#endif
.Lcheck_768_max_panel:
	MAXP_check_768
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_768_max_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_768_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_768_max_scalar
	sub $m7, $m7, 1
.Lcheck_768_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_768_max_pair
.Lcheck_768_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lcheck_768_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lcheck_768_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent FP32 maximum and denominator planes.
	ld32 $m6, $mvertex_base, $m15, 4
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lcheck_768_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lcheck_768_exp_panel:
	EXPP_check_768
	add $m3, $m3, $m8
	NEXT_check_768
	brnzdec $m7, .Lcheck_768_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lcheck_768_exp_tail:
	BEGIN_check_768
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lcheck_768_exp_odd
	sub $m7, $m7, 1
.Lcheck_768_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_768
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_768_exp_pair
.Lcheck_768_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_768_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_768_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_768_zero_panels_setup
	sub $m7, $m7, 1
.Lcheck_768_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_768_zero_pair
.Lcheck_768_zero_panels_setup:
	END_check_768
	NEXT_check_768
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_768_sum_ready
	sub $m7, $m7, 1
.Lcheck_768_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_768
	brnzdec $m7, .Lcheck_768_zero_panel
.Lcheck_768_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lcheck_768_row
.Lcheck_768_done:
	exitz $m15

// Three contiguous panel segments per row, with an explicit FP32 workspace.

	.macro INIT_check_768
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lcheck_768_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lcheck_768_parity_\@
.Lcheck_768_even_\@:
	setzi $m4, 0
.Lcheck_768_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_check_768
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// [maximum/sum, query row, segment] FP32 workspace.
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lcheck_768_nonempty_\@
	setzi $m10, 0
.Lcheck_768_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lcheck_768_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lcheck_768_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lcheck_768_clamped_\@
	mov $m10, $m1
.Lcheck_768_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m3, $m3, $m0
#ifdef ATTENTION_OUTPUT_F8
	// m0 is the score byte offset. Recover panel and row independently.
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	and $m1, $m0, 31
	shr $m0, $m0, 5
	mul $m0, $m0, $m5
	add $m0, $m0, $m4
	shl $m0, $m0, 5
	add $m0, $m0, $m1
#endif
	add $m2, $m2, $m0
	.endm

	.worker
	.p2align 3
.Lcheck_768_split_max:
	INIT_check_768
.Lcheck_768_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_768_done
	ROW_check_768
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lcheck_768_split_max_tail
	sub $m7, $m7, 1
.Lcheck_768_split_max_panel:
	MAXP_check_768
	add $m3, $m3, $m8
	brnzdec $m7, .Lcheck_768_split_max_panel
.Lcheck_768_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_768_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lcheck_768_split_max_odd
	sub $m7, $m7, 1
.Lcheck_768_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lcheck_768_split_max_pair
.Lcheck_768_split_max_odd:
	brz $m1, .Lcheck_768_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lcheck_768_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_768_split_max_row

	.p2align 3
.Lcheck_768_split_exp:
	CONFIG_check_768
	INIT_check_768
.Lcheck_768_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_768_done
	ROW_check_768
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lcheck_768_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 4
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lcheck_768_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lcheck_768_split_exp_tail
	sub $m7, $m7, 1
.Lcheck_768_split_exp_panel:
	EXPP_check_768
	add $m3, $m3, $m8
	NEXT_check_768
	brnzdec $m7, .Lcheck_768_split_exp_panel
.Lcheck_768_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lcheck_768_split_padding
	BEGIN_check_768
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lcheck_768_split_exp_odd
	sub $m7, $m7, 1
.Lcheck_768_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_check_768
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lcheck_768_split_exp_pair
.Lcheck_768_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lcheck_768_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lcheck_768_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lcheck_768_split_after_tail
	sub $m7, $m7, 1
.Lcheck_768_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lcheck_768_split_zero_pair
.Lcheck_768_split_after_tail:
	END_check_768
	NEXT_check_768
.Lcheck_768_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lcheck_768_split_padding_capacity
	mov $m1, $m7
.Lcheck_768_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lcheck_768_split_store_sum
	sub $m7, $m7, 1
.Lcheck_768_split_zero_panel:
	.rept (4 * SOFTMAX_ELEMENT_BYTES)
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	NEXT_check_768
	brnzdec $m7, .Lcheck_768_split_zero_panel
.Lcheck_768_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lcheck_768_split_exp_row

	.p2align 3
.Lcheck_768_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 4
	shl $m0, $m5, 2
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lcheck_768_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lcheck_768_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	ld32 $m6, $mvertex_base, $m15, 5
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lcheck_768_split_sum_row