#include "/srv/home/gc-sdk/ipu-stack/device/static_runtime.S"
#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_1
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_1
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_1
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_1
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_1
	.endr
	f16v4gacc $a2:3
	QUAD_old_1
	.endm
	.macro EXP_old_1
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_1_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_1_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_1_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_1_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_1_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_1_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_1_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_1_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_1_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_1_max_tail
	sub $m7, $m7, 1
#endif
.Lold_1_max_panel:
	MAXP_old_1
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_1_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_1_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_1_max_scalar
	sub $m7, $m7, 1
.Lold_1_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_1_max_pair
.Lold_1_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_1_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_1_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_1_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_1_exp_panel:
	EXPP_old_1
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_1_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_1_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_1_exp_odd
	sub $m7, $m7, 1
.Lold_1_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_1_exp_pair
.Lold_1_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_1_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_1_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_1_zero_panels_setup
	sub $m7, $m7, 1
.Lold_1_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_1_zero_pair
.Lold_1_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_1_sum_ready
	sub $m7, $m7, 1
.Lold_1_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_1_zero_panel
.Lold_1_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_1_row
.Lold_1_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_1
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_1_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_1_parity_\@
.Lold_1_even_\@:
	setzi $m4, 0
.Lold_1_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_1
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_1_nonempty_\@
	setzi $m10, 0
.Lold_1_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_1_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_1_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_1_clamped_\@
	mov $m10, $m1
.Lold_1_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_1_split_max:
	INIT_old_1
.Lold_1_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_1_done
	ROW_old_1
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_1_split_max_tail
	sub $m7, $m7, 1
.Lold_1_split_max_panel:
	MAXP_old_1
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_1_split_max_panel
.Lold_1_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_1_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_1_split_max_odd
	sub $m7, $m7, 1
.Lold_1_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_1_split_max_pair
.Lold_1_split_max_odd:
	brz $m1, .Lold_1_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_1_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_1_split_max_row

	.p2align 3
.Lold_1_split_exp:
	INIT_old_1
.Lold_1_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_1_done
	ROW_old_1
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_1_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_1_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_1_split_exp_tail
	sub $m7, $m7, 1
.Lold_1_split_exp_panel:
	EXPP_old_1
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_1_split_exp_panel
.Lold_1_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_1_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_1_split_exp_odd
	sub $m7, $m7, 1
.Lold_1_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_1_split_exp_pair
.Lold_1_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_1_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_1_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_1_split_after_tail
	sub $m7, $m7, 1
.Lold_1_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_1_split_zero_pair
.Lold_1_split_after_tail:
	add $m2, $m2, $m8
.Lold_1_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_1_split_padding_capacity
	mov $m1, $m7
.Lold_1_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_1_split_store_sum
	sub $m7, $m7, 1
.Lold_1_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_1_split_zero_panel
.Lold_1_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_1_split_exp_row

	.p2align 3
.Lold_1_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_1_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_1_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_1_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_1
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_1
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_1
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_1
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_1
	.endr
	f16v4gacc $a2:3
	QUAD_new_1
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_1
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_1_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_1_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_1_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_1_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_1_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_1_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_1_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_1_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_1_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_1_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_1_max_panel:
	MAXP_new_1
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_1_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_1_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_1_max_scalar
	sub $m7, $m7, 1
.Lnew_1_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_1_max_pair
.Lnew_1_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_1_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_1_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_1_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_1_exp_panel:
	EXPP_new_1
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_1_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_1_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_1_exp_odd
	sub $m7, $m7, 1
.Lnew_1_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_1_exp_pair
.Lnew_1_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_1_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_1_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_1_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_1_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_1_zero_pair
.Lnew_1_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_1_sum_ready
	sub $m7, $m7, 1
.Lnew_1_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_1_zero_panel
.Lnew_1_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_1_row
.Lnew_1_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_1
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_1_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_1_parity_\@
.Lnew_1_even_\@:
	setzi $m4, 0
.Lnew_1_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_1
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_1_nonempty_\@
	setzi $m10, 0
.Lnew_1_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_1_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_1_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_1_clamped_\@
	mov $m10, $m1
.Lnew_1_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_1_split_max:
	INIT_new_1
.Lnew_1_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_1_done
	ROW_new_1
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_1_split_max_tail
	sub $m7, $m7, 1
.Lnew_1_split_max_panel:
	MAXP_new_1
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_1_split_max_panel
.Lnew_1_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_1_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_1_split_max_odd
	sub $m7, $m7, 1
.Lnew_1_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_1_split_max_pair
.Lnew_1_split_max_odd:
	brz $m1, .Lnew_1_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_1_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_1_split_max_row

	.p2align 3
.Lnew_1_split_exp:
	INIT_new_1
.Lnew_1_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_1_done
	ROW_new_1
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_1_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_1_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_1_split_exp_tail
	sub $m7, $m7, 1
.Lnew_1_split_exp_panel:
	EXPP_new_1
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_1_split_exp_panel
.Lnew_1_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_1_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_1_split_exp_odd
	sub $m7, $m7, 1
.Lnew_1_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_1_split_exp_pair
.Lnew_1_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_1_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_1_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_1_split_after_tail
	sub $m7, $m7, 1
.Lnew_1_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_1_split_zero_pair
.Lnew_1_split_after_tail:
	add $m2, $m2, $m8
.Lnew_1_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_1_split_padding_capacity
	mov $m1, $m7
.Lnew_1_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_1_split_store_sum
	sub $m7, $m7, 1
.Lnew_1_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_1_split_zero_panel
.Lnew_1_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_1_split_exp_row

	.p2align 3
.Lnew_1_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_1_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_1_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_1_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_2
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_2
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_2
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_2
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_2
	.endr
	f16v4gacc $a2:3
	QUAD_old_2
	.endm
	.macro EXP_old_2
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_2_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_2_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_2_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_2_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_2_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_2_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_2_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_2_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_2_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_2_max_tail
	sub $m7, $m7, 1
#endif
.Lold_2_max_panel:
	MAXP_old_2
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_2_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_2_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_2_max_scalar
	sub $m7, $m7, 1
.Lold_2_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_2_max_pair
.Lold_2_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_2_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_2_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_2_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_2_exp_panel:
	EXPP_old_2
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_2_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_2_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_2_exp_odd
	sub $m7, $m7, 1
.Lold_2_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_2_exp_pair
.Lold_2_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_2_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_2_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_2_zero_panels_setup
	sub $m7, $m7, 1
.Lold_2_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_2_zero_pair
.Lold_2_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_2_sum_ready
	sub $m7, $m7, 1
.Lold_2_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_2_zero_panel
.Lold_2_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_2_row
.Lold_2_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_2
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_2_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_2_parity_\@
.Lold_2_even_\@:
	setzi $m4, 0
.Lold_2_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_2
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_2_nonempty_\@
	setzi $m10, 0
.Lold_2_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_2_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_2_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_2_clamped_\@
	mov $m10, $m1
.Lold_2_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_2_split_max:
	INIT_old_2
.Lold_2_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_2_done
	ROW_old_2
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_2_split_max_tail
	sub $m7, $m7, 1
.Lold_2_split_max_panel:
	MAXP_old_2
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_2_split_max_panel
.Lold_2_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_2_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_2_split_max_odd
	sub $m7, $m7, 1
.Lold_2_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_2_split_max_pair
.Lold_2_split_max_odd:
	brz $m1, .Lold_2_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_2_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_2_split_max_row

	.p2align 3
.Lold_2_split_exp:
	INIT_old_2
.Lold_2_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_2_done
	ROW_old_2
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_2_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_2_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_2_split_exp_tail
	sub $m7, $m7, 1
.Lold_2_split_exp_panel:
	EXPP_old_2
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_2_split_exp_panel
.Lold_2_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_2_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_2_split_exp_odd
	sub $m7, $m7, 1
.Lold_2_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_2_split_exp_pair
.Lold_2_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_2_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_2_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_2_split_after_tail
	sub $m7, $m7, 1
.Lold_2_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_2_split_zero_pair
.Lold_2_split_after_tail:
	add $m2, $m2, $m8
.Lold_2_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_2_split_padding_capacity
	mov $m1, $m7
.Lold_2_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_2_split_store_sum
	sub $m7, $m7, 1
.Lold_2_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_2_split_zero_panel
.Lold_2_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_2_split_exp_row

	.p2align 3
.Lold_2_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_2_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_2_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_2_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_2
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_2
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_2
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_2
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_2
	.endr
	f16v4gacc $a2:3
	QUAD_new_2
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_2
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_2_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_2_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_2_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_2_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_2_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_2_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_2_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_2_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_2_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_2_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_2_max_panel:
	MAXP_new_2
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_2_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_2_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_2_max_scalar
	sub $m7, $m7, 1
.Lnew_2_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_2_max_pair
.Lnew_2_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_2_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_2_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_2_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_2_exp_panel:
	EXPP_new_2
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_2_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_2_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_2_exp_odd
	sub $m7, $m7, 1
.Lnew_2_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_2_exp_pair
.Lnew_2_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_2_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_2_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_2_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_2_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_2_zero_pair
.Lnew_2_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_2_sum_ready
	sub $m7, $m7, 1
.Lnew_2_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_2_zero_panel
.Lnew_2_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_2_row
.Lnew_2_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_2
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_2_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_2_parity_\@
.Lnew_2_even_\@:
	setzi $m4, 0
.Lnew_2_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_2
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_2_nonempty_\@
	setzi $m10, 0
.Lnew_2_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_2_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_2_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_2_clamped_\@
	mov $m10, $m1
.Lnew_2_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_2_split_max:
	INIT_new_2
.Lnew_2_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_2_done
	ROW_new_2
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_2_split_max_tail
	sub $m7, $m7, 1
.Lnew_2_split_max_panel:
	MAXP_new_2
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_2_split_max_panel
.Lnew_2_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_2_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_2_split_max_odd
	sub $m7, $m7, 1
.Lnew_2_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_2_split_max_pair
.Lnew_2_split_max_odd:
	brz $m1, .Lnew_2_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_2_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_2_split_max_row

	.p2align 3
.Lnew_2_split_exp:
	INIT_new_2
.Lnew_2_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_2_done
	ROW_new_2
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_2_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_2_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_2_split_exp_tail
	sub $m7, $m7, 1
.Lnew_2_split_exp_panel:
	EXPP_new_2
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_2_split_exp_panel
.Lnew_2_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_2_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_2_split_exp_odd
	sub $m7, $m7, 1
.Lnew_2_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_2_split_exp_pair
.Lnew_2_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_2_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_2_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_2_split_after_tail
	sub $m7, $m7, 1
.Lnew_2_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_2_split_zero_pair
.Lnew_2_split_after_tail:
	add $m2, $m2, $m8
.Lnew_2_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_2_split_padding_capacity
	mov $m1, $m7
.Lnew_2_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_2_split_store_sum
	sub $m7, $m7, 1
.Lnew_2_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_2_split_zero_panel
.Lnew_2_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_2_split_exp_row

	.p2align 3
.Lnew_2_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_2_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_2_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_2_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_15
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_15
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_15
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_15
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_15
	.endr
	f16v4gacc $a2:3
	QUAD_old_15
	.endm
	.macro EXP_old_15
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_15_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_15_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_15_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_15_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_15_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_15_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_15_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_15_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_15_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_15_max_tail
	sub $m7, $m7, 1
#endif
.Lold_15_max_panel:
	MAXP_old_15
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_15_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_15_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_15_max_scalar
	sub $m7, $m7, 1
.Lold_15_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_15_max_pair
.Lold_15_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_15_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_15_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_15_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_15_exp_panel:
	EXPP_old_15
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_15_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_15_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_15_exp_odd
	sub $m7, $m7, 1
.Lold_15_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_15_exp_pair
.Lold_15_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_15_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_15_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_15_zero_panels_setup
	sub $m7, $m7, 1
.Lold_15_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_15_zero_pair
.Lold_15_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_15_sum_ready
	sub $m7, $m7, 1
.Lold_15_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_15_zero_panel
.Lold_15_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_15_row
.Lold_15_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_15
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_15_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_15_parity_\@
.Lold_15_even_\@:
	setzi $m4, 0
.Lold_15_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_15
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_15_nonempty_\@
	setzi $m10, 0
.Lold_15_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_15_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_15_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_15_clamped_\@
	mov $m10, $m1
.Lold_15_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_15_split_max:
	INIT_old_15
.Lold_15_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_15_done
	ROW_old_15
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_15_split_max_tail
	sub $m7, $m7, 1
.Lold_15_split_max_panel:
	MAXP_old_15
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_15_split_max_panel
.Lold_15_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_15_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_15_split_max_odd
	sub $m7, $m7, 1
.Lold_15_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_15_split_max_pair
.Lold_15_split_max_odd:
	brz $m1, .Lold_15_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_15_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_15_split_max_row

	.p2align 3
.Lold_15_split_exp:
	INIT_old_15
.Lold_15_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_15_done
	ROW_old_15
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_15_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_15_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_15_split_exp_tail
	sub $m7, $m7, 1
.Lold_15_split_exp_panel:
	EXPP_old_15
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_15_split_exp_panel
.Lold_15_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_15_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_15_split_exp_odd
	sub $m7, $m7, 1
.Lold_15_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_15_split_exp_pair
.Lold_15_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_15_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_15_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_15_split_after_tail
	sub $m7, $m7, 1
.Lold_15_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_15_split_zero_pair
.Lold_15_split_after_tail:
	add $m2, $m2, $m8
.Lold_15_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_15_split_padding_capacity
	mov $m1, $m7
.Lold_15_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_15_split_store_sum
	sub $m7, $m7, 1
.Lold_15_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_15_split_zero_panel
.Lold_15_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_15_split_exp_row

	.p2align 3
.Lold_15_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_15_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_15_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_15_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_15
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_15
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_15
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_15
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_15
	.endr
	f16v4gacc $a2:3
	QUAD_new_15
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_15
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_15_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_15_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_15_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_15_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_15_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_15_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_15_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_15_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_15_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_15_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_15_max_panel:
	MAXP_new_15
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_15_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_15_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_15_max_scalar
	sub $m7, $m7, 1
.Lnew_15_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_15_max_pair
.Lnew_15_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_15_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_15_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_15_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_15_exp_panel:
	EXPP_new_15
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_15_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_15_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_15_exp_odd
	sub $m7, $m7, 1
.Lnew_15_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_15_exp_pair
.Lnew_15_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_15_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_15_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_15_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_15_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_15_zero_pair
.Lnew_15_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_15_sum_ready
	sub $m7, $m7, 1
.Lnew_15_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_15_zero_panel
.Lnew_15_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_15_row
.Lnew_15_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_15
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_15_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_15_parity_\@
.Lnew_15_even_\@:
	setzi $m4, 0
.Lnew_15_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_15
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_15_nonempty_\@
	setzi $m10, 0
.Lnew_15_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_15_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_15_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_15_clamped_\@
	mov $m10, $m1
.Lnew_15_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_15_split_max:
	INIT_new_15
.Lnew_15_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_15_done
	ROW_new_15
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_15_split_max_tail
	sub $m7, $m7, 1
.Lnew_15_split_max_panel:
	MAXP_new_15
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_15_split_max_panel
.Lnew_15_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_15_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_15_split_max_odd
	sub $m7, $m7, 1
.Lnew_15_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_15_split_max_pair
.Lnew_15_split_max_odd:
	brz $m1, .Lnew_15_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_15_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_15_split_max_row

	.p2align 3
.Lnew_15_split_exp:
	INIT_new_15
.Lnew_15_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_15_done
	ROW_new_15
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_15_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_15_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_15_split_exp_tail
	sub $m7, $m7, 1
.Lnew_15_split_exp_panel:
	EXPP_new_15
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_15_split_exp_panel
.Lnew_15_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_15_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_15_split_exp_odd
	sub $m7, $m7, 1
.Lnew_15_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_15_split_exp_pair
.Lnew_15_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_15_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_15_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_15_split_after_tail
	sub $m7, $m7, 1
.Lnew_15_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_15_split_zero_pair
.Lnew_15_split_after_tail:
	add $m2, $m2, $m8
.Lnew_15_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_15_split_padding_capacity
	mov $m1, $m7
.Lnew_15_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_15_split_store_sum
	sub $m7, $m7, 1
.Lnew_15_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_15_split_zero_panel
.Lnew_15_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_15_split_exp_row

	.p2align 3
.Lnew_15_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_15_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_15_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_15_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_16
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_16
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_16
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_16
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_16
	.endr
	f16v4gacc $a2:3
	QUAD_old_16
	.endm
	.macro EXP_old_16
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_16_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_16_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_16_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_16_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_16_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_16_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_16_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_16_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_16_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_16_max_tail
	sub $m7, $m7, 1
#endif
.Lold_16_max_panel:
	MAXP_old_16
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_16_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_16_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_16_max_scalar
	sub $m7, $m7, 1
.Lold_16_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_16_max_pair
.Lold_16_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_16_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_16_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_16_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_16_exp_panel:
	EXPP_old_16
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_16_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_16_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_16_exp_odd
	sub $m7, $m7, 1
.Lold_16_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_16_exp_pair
.Lold_16_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_16_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_16_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_16_zero_panels_setup
	sub $m7, $m7, 1
.Lold_16_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_16_zero_pair
.Lold_16_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_16_sum_ready
	sub $m7, $m7, 1
.Lold_16_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_16_zero_panel
.Lold_16_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_16_row
.Lold_16_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_16
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_16_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_16_parity_\@
.Lold_16_even_\@:
	setzi $m4, 0
.Lold_16_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_16
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_16_nonempty_\@
	setzi $m10, 0
.Lold_16_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_16_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_16_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_16_clamped_\@
	mov $m10, $m1
.Lold_16_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_16_split_max:
	INIT_old_16
.Lold_16_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_16_done
	ROW_old_16
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_16_split_max_tail
	sub $m7, $m7, 1
.Lold_16_split_max_panel:
	MAXP_old_16
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_16_split_max_panel
.Lold_16_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_16_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_16_split_max_odd
	sub $m7, $m7, 1
.Lold_16_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_16_split_max_pair
.Lold_16_split_max_odd:
	brz $m1, .Lold_16_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_16_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_16_split_max_row

	.p2align 3
.Lold_16_split_exp:
	INIT_old_16
.Lold_16_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_16_done
	ROW_old_16
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_16_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_16_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_16_split_exp_tail
	sub $m7, $m7, 1
.Lold_16_split_exp_panel:
	EXPP_old_16
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_16_split_exp_panel
.Lold_16_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_16_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_16_split_exp_odd
	sub $m7, $m7, 1
.Lold_16_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_16_split_exp_pair
.Lold_16_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_16_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_16_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_16_split_after_tail
	sub $m7, $m7, 1
.Lold_16_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_16_split_zero_pair
.Lold_16_split_after_tail:
	add $m2, $m2, $m8
.Lold_16_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_16_split_padding_capacity
	mov $m1, $m7
.Lold_16_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_16_split_store_sum
	sub $m7, $m7, 1
.Lold_16_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_16_split_zero_panel
.Lold_16_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_16_split_exp_row

	.p2align 3
.Lold_16_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_16_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_16_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_16_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_16
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_16
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_16
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_16
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_16
	.endr
	f16v4gacc $a2:3
	QUAD_new_16
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_16
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_16_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_16_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_16_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_16_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_16_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_16_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_16_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_16_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_16_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_16_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_16_max_panel:
	MAXP_new_16
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_16_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_16_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_16_max_scalar
	sub $m7, $m7, 1
.Lnew_16_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_16_max_pair
.Lnew_16_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_16_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_16_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_16_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_16_exp_panel:
	EXPP_new_16
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_16_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_16_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_16_exp_odd
	sub $m7, $m7, 1
.Lnew_16_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_16_exp_pair
.Lnew_16_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_16_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_16_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_16_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_16_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_16_zero_pair
.Lnew_16_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_16_sum_ready
	sub $m7, $m7, 1
.Lnew_16_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_16_zero_panel
.Lnew_16_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_16_row
.Lnew_16_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_16
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_16_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_16_parity_\@
.Lnew_16_even_\@:
	setzi $m4, 0
.Lnew_16_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_16
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_16_nonempty_\@
	setzi $m10, 0
.Lnew_16_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_16_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_16_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_16_clamped_\@
	mov $m10, $m1
.Lnew_16_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_16_split_max:
	INIT_new_16
.Lnew_16_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_16_done
	ROW_new_16
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_16_split_max_tail
	sub $m7, $m7, 1
.Lnew_16_split_max_panel:
	MAXP_new_16
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_16_split_max_panel
.Lnew_16_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_16_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_16_split_max_odd
	sub $m7, $m7, 1
.Lnew_16_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_16_split_max_pair
.Lnew_16_split_max_odd:
	brz $m1, .Lnew_16_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_16_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_16_split_max_row

	.p2align 3
.Lnew_16_split_exp:
	INIT_new_16
.Lnew_16_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_16_done
	ROW_new_16
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_16_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_16_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_16_split_exp_tail
	sub $m7, $m7, 1
.Lnew_16_split_exp_panel:
	EXPP_new_16
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_16_split_exp_panel
.Lnew_16_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_16_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_16_split_exp_odd
	sub $m7, $m7, 1
.Lnew_16_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_16_split_exp_pair
.Lnew_16_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_16_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_16_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_16_split_after_tail
	sub $m7, $m7, 1
.Lnew_16_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_16_split_zero_pair
.Lnew_16_split_after_tail:
	add $m2, $m2, $m8
.Lnew_16_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_16_split_padding_capacity
	mov $m1, $m7
.Lnew_16_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_16_split_store_sum
	sub $m7, $m7, 1
.Lnew_16_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_16_split_zero_panel
.Lnew_16_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_16_split_exp_row

	.p2align 3
.Lnew_16_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_16_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_16_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_16_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_17
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_17
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_17
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_17
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_17
	.endr
	f16v4gacc $a2:3
	QUAD_old_17
	.endm
	.macro EXP_old_17
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_17_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_17_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_17_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_17_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_17_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_17_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_17_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_17_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_17_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_17_max_tail
	sub $m7, $m7, 1
#endif
.Lold_17_max_panel:
	MAXP_old_17
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_17_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_17_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_17_max_scalar
	sub $m7, $m7, 1
.Lold_17_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_17_max_pair
.Lold_17_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_17_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_17_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_17_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_17_exp_panel:
	EXPP_old_17
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_17_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_17_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_17_exp_odd
	sub $m7, $m7, 1
.Lold_17_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_17_exp_pair
.Lold_17_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_17_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_17_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_17_zero_panels_setup
	sub $m7, $m7, 1
.Lold_17_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_17_zero_pair
.Lold_17_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_17_sum_ready
	sub $m7, $m7, 1
.Lold_17_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_17_zero_panel
.Lold_17_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_17_row
.Lold_17_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_17
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_17_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_17_parity_\@
.Lold_17_even_\@:
	setzi $m4, 0
.Lold_17_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_17
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_17_nonempty_\@
	setzi $m10, 0
.Lold_17_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_17_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_17_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_17_clamped_\@
	mov $m10, $m1
.Lold_17_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_17_split_max:
	INIT_old_17
.Lold_17_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_17_done
	ROW_old_17
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_17_split_max_tail
	sub $m7, $m7, 1
.Lold_17_split_max_panel:
	MAXP_old_17
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_17_split_max_panel
.Lold_17_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_17_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_17_split_max_odd
	sub $m7, $m7, 1
.Lold_17_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_17_split_max_pair
.Lold_17_split_max_odd:
	brz $m1, .Lold_17_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_17_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_17_split_max_row

	.p2align 3
.Lold_17_split_exp:
	INIT_old_17
.Lold_17_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_17_done
	ROW_old_17
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_17_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_17_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_17_split_exp_tail
	sub $m7, $m7, 1
.Lold_17_split_exp_panel:
	EXPP_old_17
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_17_split_exp_panel
.Lold_17_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_17_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_17_split_exp_odd
	sub $m7, $m7, 1
.Lold_17_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_17_split_exp_pair
.Lold_17_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_17_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_17_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_17_split_after_tail
	sub $m7, $m7, 1
.Lold_17_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_17_split_zero_pair
.Lold_17_split_after_tail:
	add $m2, $m2, $m8
.Lold_17_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_17_split_padding_capacity
	mov $m1, $m7
.Lold_17_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_17_split_store_sum
	sub $m7, $m7, 1
.Lold_17_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_17_split_zero_panel
.Lold_17_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_17_split_exp_row

	.p2align 3
.Lold_17_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_17_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_17_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_17_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_17
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_17
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_17
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_17
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_17
	.endr
	f16v4gacc $a2:3
	QUAD_new_17
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_17
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_17_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_17_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_17_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_17_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_17_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_17_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_17_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_17_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_17_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_17_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_17_max_panel:
	MAXP_new_17
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_17_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_17_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_17_max_scalar
	sub $m7, $m7, 1
.Lnew_17_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_17_max_pair
.Lnew_17_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_17_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_17_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_17_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_17_exp_panel:
	EXPP_new_17
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_17_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_17_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_17_exp_odd
	sub $m7, $m7, 1
.Lnew_17_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_17_exp_pair
.Lnew_17_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_17_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_17_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_17_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_17_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_17_zero_pair
.Lnew_17_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_17_sum_ready
	sub $m7, $m7, 1
.Lnew_17_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_17_zero_panel
.Lnew_17_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_17_row
.Lnew_17_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_17
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_17_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_17_parity_\@
.Lnew_17_even_\@:
	setzi $m4, 0
.Lnew_17_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_17
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_17_nonempty_\@
	setzi $m10, 0
.Lnew_17_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_17_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_17_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_17_clamped_\@
	mov $m10, $m1
.Lnew_17_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_17_split_max:
	INIT_new_17
.Lnew_17_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_17_done
	ROW_new_17
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_17_split_max_tail
	sub $m7, $m7, 1
.Lnew_17_split_max_panel:
	MAXP_new_17
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_17_split_max_panel
.Lnew_17_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_17_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_17_split_max_odd
	sub $m7, $m7, 1
.Lnew_17_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_17_split_max_pair
.Lnew_17_split_max_odd:
	brz $m1, .Lnew_17_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_17_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_17_split_max_row

	.p2align 3
.Lnew_17_split_exp:
	INIT_new_17
.Lnew_17_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_17_done
	ROW_new_17
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_17_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_17_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_17_split_exp_tail
	sub $m7, $m7, 1
.Lnew_17_split_exp_panel:
	EXPP_new_17
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_17_split_exp_panel
.Lnew_17_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_17_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_17_split_exp_odd
	sub $m7, $m7, 1
.Lnew_17_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_17_split_exp_pair
.Lnew_17_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_17_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_17_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_17_split_after_tail
	sub $m7, $m7, 1
.Lnew_17_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_17_split_zero_pair
.Lnew_17_split_after_tail:
	add $m2, $m2, $m8
.Lnew_17_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_17_split_padding_capacity
	mov $m1, $m7
.Lnew_17_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_17_split_store_sum
	sub $m7, $m7, 1
.Lnew_17_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_17_split_zero_panel
.Lnew_17_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_17_split_exp_row

	.p2align 3
.Lnew_17_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_17_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_17_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_17_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_25
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_25
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_25
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_25
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_25
	.endr
	f16v4gacc $a2:3
	QUAD_old_25
	.endm
	.macro EXP_old_25
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_25_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_25_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_25_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_25_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_25_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_25_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_25_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_25_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_25_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_25_max_tail
	sub $m7, $m7, 1
#endif
.Lold_25_max_panel:
	MAXP_old_25
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_25_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_25_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_25_max_scalar
	sub $m7, $m7, 1
.Lold_25_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_25_max_pair
.Lold_25_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_25_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_25_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_25_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_25_exp_panel:
	EXPP_old_25
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_25_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_25_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_25_exp_odd
	sub $m7, $m7, 1
.Lold_25_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_25
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_25_exp_pair
.Lold_25_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_25_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_25_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_25_zero_panels_setup
	sub $m7, $m7, 1
.Lold_25_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_25_zero_pair
.Lold_25_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_25_sum_ready
	sub $m7, $m7, 1
.Lold_25_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_25_zero_panel
.Lold_25_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_25_row
.Lold_25_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_25
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_25_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_25_parity_\@
.Lold_25_even_\@:
	setzi $m4, 0
.Lold_25_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_25
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_25_nonempty_\@
	setzi $m10, 0
.Lold_25_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_25_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_25_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_25_clamped_\@
	mov $m10, $m1
.Lold_25_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_25_split_max:
	INIT_old_25
.Lold_25_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_25_done
	ROW_old_25
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_25_split_max_tail
	sub $m7, $m7, 1
.Lold_25_split_max_panel:
	MAXP_old_25
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_25_split_max_panel
.Lold_25_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_25_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_25_split_max_odd
	sub $m7, $m7, 1
.Lold_25_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_25_split_max_pair
.Lold_25_split_max_odd:
	brz $m1, .Lold_25_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_25_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_25_split_max_row

	.p2align 3
.Lold_25_split_exp:
	INIT_old_25
.Lold_25_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_25_done
	ROW_old_25
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_25_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_25_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_25_split_exp_tail
	sub $m7, $m7, 1
.Lold_25_split_exp_panel:
	EXPP_old_25
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_25_split_exp_panel
.Lold_25_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_25_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_25_split_exp_odd
	sub $m7, $m7, 1
.Lold_25_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_25
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_25_split_exp_pair
.Lold_25_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_25_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_25_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_25_split_after_tail
	sub $m7, $m7, 1
.Lold_25_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_25_split_zero_pair
.Lold_25_split_after_tail:
	add $m2, $m2, $m8
.Lold_25_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_25_split_padding_capacity
	mov $m1, $m7
.Lold_25_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_25_split_store_sum
	sub $m7, $m7, 1
.Lold_25_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_25_split_zero_panel
.Lold_25_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_25_split_exp_row

	.p2align 3
.Lold_25_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_25_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_25_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_25_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_25
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_25
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_25
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_25
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_25
	.endr
	f16v4gacc $a2:3
	QUAD_new_25
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_25
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_25_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_25_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_25_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_25_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_25_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_25_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_25_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_25_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_25_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_25_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_25_max_panel:
	MAXP_new_25
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_25_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_25_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_25_max_scalar
	sub $m7, $m7, 1
.Lnew_25_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_25_max_pair
.Lnew_25_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_25_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_25_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_25_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_25_exp_panel:
	EXPP_new_25
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_25_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_25_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_25_exp_odd
	sub $m7, $m7, 1
.Lnew_25_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_25
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_25_exp_pair
.Lnew_25_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_25_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_25_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_25_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_25_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_25_zero_pair
.Lnew_25_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_25_sum_ready
	sub $m7, $m7, 1
.Lnew_25_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_25_zero_panel
.Lnew_25_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_25_row
.Lnew_25_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_25
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_25_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_25_parity_\@
.Lnew_25_even_\@:
	setzi $m4, 0
.Lnew_25_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_25
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_25_nonempty_\@
	setzi $m10, 0
.Lnew_25_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_25_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_25_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_25_clamped_\@
	mov $m10, $m1
.Lnew_25_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_25_split_max:
	INIT_new_25
.Lnew_25_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_25_done
	ROW_new_25
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_25_split_max_tail
	sub $m7, $m7, 1
.Lnew_25_split_max_panel:
	MAXP_new_25
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_25_split_max_panel
.Lnew_25_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_25_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_25_split_max_odd
	sub $m7, $m7, 1
.Lnew_25_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_25_split_max_pair
.Lnew_25_split_max_odd:
	brz $m1, .Lnew_25_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_25_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_25_split_max_row

	.p2align 3
.Lnew_25_split_exp:
	INIT_new_25
.Lnew_25_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_25_done
	ROW_new_25
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_25_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_25_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_25_split_exp_tail
	sub $m7, $m7, 1
.Lnew_25_split_exp_panel:
	EXPP_new_25
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_25_split_exp_panel
.Lnew_25_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_25_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_25_split_exp_odd
	sub $m7, $m7, 1
.Lnew_25_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_25
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_25_split_exp_pair
.Lnew_25_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_25_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_25_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_25_split_after_tail
	sub $m7, $m7, 1
.Lnew_25_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_25_split_zero_pair
.Lnew_25_split_after_tail:
	add $m2, $m2, $m8
.Lnew_25_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_25_split_padding_capacity
	mov $m1, $m7
.Lnew_25_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_25_split_store_sum
	sub $m7, $m7, 1
.Lnew_25_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_25_split_zero_panel
.Lnew_25_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_25_split_exp_row

	.p2align 3
.Lnew_25_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_25_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_25_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_25_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_31
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_31
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_31
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_31
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_31
	.endr
	f16v4gacc $a2:3
	QUAD_old_31
	.endm
	.macro EXP_old_31
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_31_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_31_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_31_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_31_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_31_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_31_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_31_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_31_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_31_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_31_max_tail
	sub $m7, $m7, 1
#endif
.Lold_31_max_panel:
	MAXP_old_31
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_31_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_31_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_31_max_scalar
	sub $m7, $m7, 1
.Lold_31_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_31_max_pair
.Lold_31_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_31_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_31_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_31_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_31_exp_panel:
	EXPP_old_31
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_31_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_31_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_31_exp_odd
	sub $m7, $m7, 1
.Lold_31_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_31_exp_pair
.Lold_31_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_31_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_31_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_31_zero_panels_setup
	sub $m7, $m7, 1
.Lold_31_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_31_zero_pair
.Lold_31_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_31_sum_ready
	sub $m7, $m7, 1
.Lold_31_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_31_zero_panel
.Lold_31_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_31_row
.Lold_31_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_31
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_31_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_31_parity_\@
.Lold_31_even_\@:
	setzi $m4, 0
.Lold_31_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_31
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_31_nonempty_\@
	setzi $m10, 0
.Lold_31_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_31_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_31_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_31_clamped_\@
	mov $m10, $m1
.Lold_31_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_31_split_max:
	INIT_old_31
.Lold_31_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_31_done
	ROW_old_31
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_31_split_max_tail
	sub $m7, $m7, 1
.Lold_31_split_max_panel:
	MAXP_old_31
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_31_split_max_panel
.Lold_31_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_31_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_31_split_max_odd
	sub $m7, $m7, 1
.Lold_31_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_31_split_max_pair
.Lold_31_split_max_odd:
	brz $m1, .Lold_31_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_31_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_31_split_max_row

	.p2align 3
.Lold_31_split_exp:
	INIT_old_31
.Lold_31_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_31_done
	ROW_old_31
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_31_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_31_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_31_split_exp_tail
	sub $m7, $m7, 1
.Lold_31_split_exp_panel:
	EXPP_old_31
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_31_split_exp_panel
.Lold_31_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_31_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_31_split_exp_odd
	sub $m7, $m7, 1
.Lold_31_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_31_split_exp_pair
.Lold_31_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_31_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_31_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_31_split_after_tail
	sub $m7, $m7, 1
.Lold_31_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_31_split_zero_pair
.Lold_31_split_after_tail:
	add $m2, $m2, $m8
.Lold_31_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_31_split_padding_capacity
	mov $m1, $m7
.Lold_31_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_31_split_store_sum
	sub $m7, $m7, 1
.Lold_31_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_31_split_zero_panel
.Lold_31_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_31_split_exp_row

	.p2align 3
.Lold_31_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_31_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_31_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_31_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_31
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_31
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_31
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_31
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_31
	.endr
	f16v4gacc $a2:3
	QUAD_new_31
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_31
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_31_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_31_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_31_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_31_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_31_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_31_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_31_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_31_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_31_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_31_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_31_max_panel:
	MAXP_new_31
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_31_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_31_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_31_max_scalar
	sub $m7, $m7, 1
.Lnew_31_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_31_max_pair
.Lnew_31_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_31_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_31_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_31_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_31_exp_panel:
	EXPP_new_31
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_31_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_31_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_31_exp_odd
	sub $m7, $m7, 1
.Lnew_31_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_31_exp_pair
.Lnew_31_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_31_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_31_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_31_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_31_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_31_zero_pair
.Lnew_31_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_31_sum_ready
	sub $m7, $m7, 1
.Lnew_31_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_31_zero_panel
.Lnew_31_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_31_row
.Lnew_31_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_31
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_31_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_31_parity_\@
.Lnew_31_even_\@:
	setzi $m4, 0
.Lnew_31_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_31
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_31_nonempty_\@
	setzi $m10, 0
.Lnew_31_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_31_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_31_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_31_clamped_\@
	mov $m10, $m1
.Lnew_31_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_31_split_max:
	INIT_new_31
.Lnew_31_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_31_done
	ROW_new_31
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_31_split_max_tail
	sub $m7, $m7, 1
.Lnew_31_split_max_panel:
	MAXP_new_31
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_31_split_max_panel
.Lnew_31_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_31_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_31_split_max_odd
	sub $m7, $m7, 1
.Lnew_31_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_31_split_max_pair
.Lnew_31_split_max_odd:
	brz $m1, .Lnew_31_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_31_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_31_split_max_row

	.p2align 3
.Lnew_31_split_exp:
	INIT_new_31
.Lnew_31_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_31_done
	ROW_new_31
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_31_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_31_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_31_split_exp_tail
	sub $m7, $m7, 1
.Lnew_31_split_exp_panel:
	EXPP_new_31
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_31_split_exp_panel
.Lnew_31_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_31_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_31_split_exp_odd
	sub $m7, $m7, 1
.Lnew_31_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_31_split_exp_pair
.Lnew_31_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_31_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_31_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_31_split_after_tail
	sub $m7, $m7, 1
.Lnew_31_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_31_split_zero_pair
.Lnew_31_split_after_tail:
	add $m2, $m2, $m8
.Lnew_31_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_31_split_padding_capacity
	mov $m1, $m7
.Lnew_31_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_31_split_store_sum
	sub $m7, $m7, 1
.Lnew_31_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_31_split_zero_panel
.Lnew_31_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_31_split_exp_row

	.p2align 3
.Lnew_31_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_31_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_31_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_31_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 64
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_64
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_64
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_64
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_64
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_64
	.endr
	f16v4gacc $a2:3
	QUAD_old_64
	.endm
	.macro EXP_old_64
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_64_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_64_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_64_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_64_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_64_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_64_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_64_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_64_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_64_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_64_max_tail
	sub $m7, $m7, 1
#endif
.Lold_64_max_panel:
	MAXP_old_64
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_64_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_64_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_64_max_scalar
	sub $m7, $m7, 1
.Lold_64_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_64_max_pair
.Lold_64_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_64_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_64_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_64_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_64_exp_panel:
	EXPP_old_64
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_64_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_64_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_64_exp_odd
	sub $m7, $m7, 1
.Lold_64_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_64_exp_pair
.Lold_64_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_64_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_64_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_64_zero_panels_setup
	sub $m7, $m7, 1
.Lold_64_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_64_zero_pair
.Lold_64_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_64_sum_ready
	sub $m7, $m7, 1
.Lold_64_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_64_zero_panel
.Lold_64_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_64_row
.Lold_64_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_64
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_64_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_64_parity_\@
.Lold_64_even_\@:
	setzi $m4, 0
.Lold_64_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_64
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_64_nonempty_\@
	setzi $m10, 0
.Lold_64_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_64_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_64_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_64_clamped_\@
	mov $m10, $m1
.Lold_64_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_64_split_max:
	INIT_old_64
.Lold_64_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_64_done
	ROW_old_64
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_64_split_max_tail
	sub $m7, $m7, 1
.Lold_64_split_max_panel:
	MAXP_old_64
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_64_split_max_panel
.Lold_64_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_64_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_64_split_max_odd
	sub $m7, $m7, 1
.Lold_64_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_64_split_max_pair
.Lold_64_split_max_odd:
	brz $m1, .Lold_64_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_64_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_64_split_max_row

	.p2align 3
.Lold_64_split_exp:
	INIT_old_64
.Lold_64_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_64_done
	ROW_old_64
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_64_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_64_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_64_split_exp_tail
	sub $m7, $m7, 1
.Lold_64_split_exp_panel:
	EXPP_old_64
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_64_split_exp_panel
.Lold_64_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_64_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_64_split_exp_odd
	sub $m7, $m7, 1
.Lold_64_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_64_split_exp_pair
.Lold_64_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_64_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_64_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_64_split_after_tail
	sub $m7, $m7, 1
.Lold_64_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_64_split_zero_pair
.Lold_64_split_after_tail:
	add $m2, $m2, $m8
.Lold_64_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_64_split_padding_capacity
	mov $m1, $m7
.Lold_64_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_64_split_store_sum
	sub $m7, $m7, 1
.Lold_64_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_64_split_zero_panel
.Lold_64_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_64_split_exp_row

	.p2align 3
.Lold_64_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_64_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_64_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_64_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 64
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_64
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_64
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_64
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_64
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_64
	.endr
	f16v4gacc $a2:3
	QUAD_new_64
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_64
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_64_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_64_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_64_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_64_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_64_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_64_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_64_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_64_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_64_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_64_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_64_max_panel:
	MAXP_new_64
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_64_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_64_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_64_max_scalar
	sub $m7, $m7, 1
.Lnew_64_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_64_max_pair
.Lnew_64_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_64_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_64_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_64_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_64_exp_panel:
	EXPP_new_64
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_64_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_64_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_64_exp_odd
	sub $m7, $m7, 1
.Lnew_64_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_64_exp_pair
.Lnew_64_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_64_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_64_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_64_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_64_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_64_zero_pair
.Lnew_64_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_64_sum_ready
	sub $m7, $m7, 1
.Lnew_64_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_64_zero_panel
.Lnew_64_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_64_row
.Lnew_64_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_64
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_64_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_64_parity_\@
.Lnew_64_even_\@:
	setzi $m4, 0
.Lnew_64_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_64
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_64_nonempty_\@
	setzi $m10, 0
.Lnew_64_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_64_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_64_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_64_clamped_\@
	mov $m10, $m1
.Lnew_64_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_64_split_max:
	INIT_new_64
.Lnew_64_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_64_done
	ROW_new_64
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_64_split_max_tail
	sub $m7, $m7, 1
.Lnew_64_split_max_panel:
	MAXP_new_64
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_64_split_max_panel
.Lnew_64_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_64_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_64_split_max_odd
	sub $m7, $m7, 1
.Lnew_64_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_64_split_max_pair
.Lnew_64_split_max_odd:
	brz $m1, .Lnew_64_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_64_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_64_split_max_row

	.p2align 3
.Lnew_64_split_exp:
	INIT_new_64
.Lnew_64_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_64_done
	ROW_new_64
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_64_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_64_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_64_split_exp_tail
	sub $m7, $m7, 1
.Lnew_64_split_exp_panel:
	EXPP_new_64
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_64_split_exp_panel
.Lnew_64_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_64_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_64_split_exp_odd
	sub $m7, $m7, 1
.Lnew_64_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_64_split_exp_pair
.Lnew_64_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_64_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_64_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_64_split_after_tail
	sub $m7, $m7, 1
.Lnew_64_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_64_split_zero_pair
.Lnew_64_split_after_tail:
	add $m2, $m2, $m8
.Lnew_64_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_64_split_padding_capacity
	mov $m1, $m7
.Lnew_64_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_64_split_store_sum
	sub $m7, $m7, 1
.Lnew_64_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_64_split_zero_panel
.Lnew_64_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_64_split_exp_row

	.p2align 3
.Lnew_64_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_64_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_64_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_64_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 80
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_65
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_old_65
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_old_65
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a2 }
	f32v2add $a6:7, $a6:7, $a0:1
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2tof32 $a0:1, $a3 }
	f32v2add $a6:7, $a6:7, $a0:1
	.endm
	.macro EXPP_old_65
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_old_65
	.endr
	f16v4gacc $a2:3
	QUAD_old_65
	.endm
	.macro EXP_old_65
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lold_65_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lold_65_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_65_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lold_65_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lold_65_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_65_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_65_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_65_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_65_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_65_max_tail
	sub $m7, $m7, 1
#endif
.Lold_65_max_panel:
	MAXP_old_65
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_65_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_65_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_65_max_scalar
	sub $m7, $m7, 1
.Lold_65_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_65_max_pair
.Lold_65_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_65_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_65_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_65_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lold_65_exp_panel:
	EXPP_old_65
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_65_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_65_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_65_exp_odd
	sub $m7, $m7, 1
.Lold_65_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_65_exp_pair
.Lold_65_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_65_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_65_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_65_zero_panels_setup
	sub $m7, $m7, 1
.Lold_65_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_65_zero_pair
.Lold_65_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_65_sum_ready
	sub $m7, $m7, 1
.Lold_65_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_65_zero_panel
.Lold_65_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_65_row
.Lold_65_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_old_65
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lold_65_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lold_65_parity_\@
.Lold_65_even_\@:
	setzi $m4, 0
.Lold_65_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_old_65
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lold_65_nonempty_\@
	setzi $m10, 0
.Lold_65_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lold_65_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lold_65_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lold_65_clamped_\@
	mov $m10, $m1
.Lold_65_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lold_65_split_max:
	INIT_old_65
.Lold_65_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_65_done
	ROW_old_65
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lold_65_split_max_tail
	sub $m7, $m7, 1
.Lold_65_split_max_panel:
	MAXP_old_65
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_65_split_max_panel
.Lold_65_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_65_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lold_65_split_max_odd
	sub $m7, $m7, 1
.Lold_65_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_65_split_max_pair
.Lold_65_split_max_odd:
	brz $m1, .Lold_65_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lold_65_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_65_split_max_row

	.p2align 3
.Lold_65_split_exp:
	INIT_old_65
.Lold_65_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_65_done
	ROW_old_65
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lold_65_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lold_65_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lold_65_split_exp_tail
	sub $m7, $m7, 1
.Lold_65_split_exp_panel:
	EXPP_old_65
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_65_split_exp_panel
.Lold_65_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lold_65_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lold_65_split_exp_odd
	sub $m7, $m7, 1
.Lold_65_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_65_split_exp_pair
.Lold_65_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_65_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_65_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_65_split_after_tail
	sub $m7, $m7, 1
.Lold_65_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_65_split_zero_pair
.Lold_65_split_after_tail:
	add $m2, $m2, $m8
.Lold_65_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lold_65_split_padding_capacity
	mov $m1, $m7
.Lold_65_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lold_65_split_store_sum
	sub $m7, $m7, 1
.Lold_65_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_65_split_zero_panel
.Lold_65_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lold_65_split_exp_row

	.p2align 3
.Lold_65_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lold_65_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_65_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lold_65_split_sum_row

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 80
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_65
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"
// Shared panel arithmetic for whole-row and segmented softmax workers.
	.macro MAXP_new_65
	mov $a7, $a6
	ld64step $a0:1, $mzero, $m3+=, 1
	.rept 3
	{ ld64step $a0:1, $mzero, $m3+=, 1
	  f16v4max $a6:7, $a6:7, $a0:1 }
	.endr
	f16v4max $a6:7, $a6:7, $a0:1
	f16v2max $a6, $a6, $a7
	.endm
	.macro QUAD_new_65
	f16v2exp $a2, $a2
	f16v2exp $a3, $a3
	{ st32step $a2, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a2 }
	{ st32step $a3, $mzero, $m2+=, 1
	  f16v2add $a6, $a6, $a3 }
	.endm
	.macro EXPP_new_65
	// Sum only eight nonnegative exponentials per FP16 lane (each <= 1),
	// then accumulate the panel in FP32. Store the unchanged exponentials.
	// Prime with zero accumulators so the discarded readout is finite.
	// MIX then returns each preceding quad while starting the next one.
	setzi $a0, 1 << CSR_W_FP_CLR__ZAACC__SHIFT
	uput $FP_CLR, $a0
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	.rept 3
	ld64step $a0:1, $mzero, $m3+=, 1
	f16v4mix $a2:3, $a0:1, $a4:5
	QUAD_new_65
	.endr
	f16v4gacc $a2:3
	QUAD_new_65
	f16v2tof32 $a0:1, $a6
	f32add $a0, $a0, $a1
	f32add $a7, $a7, $a0
	setzi $a6, 0
	.endm
	.macro EXP_new_65
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys, m6 split-row flag.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	// m6 selects the cheaper row schedule, as priced by the host cost model.
	brz $m6, .Lnew_65_whole
	add $m11, $m11, -16
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
	st32 $m5, $m11, $m15, 3
	setzi $m0, .Lnew_65_split_max
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_65_split_exp
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	setzi $m0, .Lnew_65_split_sum
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, 16
	br $m10
.Lnew_65_whole:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_65_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_65_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_65_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_65_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_65_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_65_max_panel:
	MAXP_new_65
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_65_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_65_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_65_max_scalar
	sub $m7, $m7, 1
.Lnew_65_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_65_max_pair
.Lnew_65_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_65_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_65_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	f32tof16 $a2, $a6
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_65_exp_tail
	sub $m7, $m7, 1
#endif
	// TAS holds +/- the half scale. MIX forms the difference in FP32,
	// avoiding an overflowing intermediate FP16 subtraction.
	mov $a0, $a4
	f32sub $a1, $azero, $a4
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	mov $a4, $a2
	mov $a5, $a2
.Lnew_65_exp_panel:
	EXPP_new_65
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_65_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_65_exp_tail:
	ldconst $a4, ATTENTION_SCALE_BITS
	ld32 $a5, $m6, $m15, 0
	f32sub $a5, $azero, $a5
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_65_exp_odd
	sub $m7, $m7, 1
.Lnew_65_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_65_exp_pair
.Lnew_65_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_65_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_65_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_65_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_65_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_65_zero_pair
.Lnew_65_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_65_sum_ready
	sub $m7, $m7, 1
.Lnew_65_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_65_zero_panel
.Lnew_65_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_65_row
.Lnew_65_done:
	exitz $m15

// Three contiguous panel segments per row; scratch follows maxima/sums.

	.macro INIT_new_65
	ld32 $m5, $mvertex_base, $m15, 2
	get $m9, $WSR
	and $m9, $m9, CSR_W_WSR__CTXTID_M1__MASK
	setzi $m0, 3
	cmpult $m4, $m9, $m0
	brnz $m4, .Lnew_65_even_\@
	sub $m9, $m9, 3
	setzi $m4, 1
	bri .Lnew_65_parity_\@
.Lnew_65_even_\@:
	setzi $m4, 0
.Lnew_65_parity_\@:
	add $m8, $m5, -1
	shl $m8, $m8, 5
	.endm

	.macro ROW_new_65
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	// Three FP32 maxima and three sums use the existing extra 16 halves/row.
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 8
	mul $m6, $m5, $m0
	add $m6, $m6, $m2
	setzi $m0, 3
	mul $m0, $m4, $m0
	add $m0, $m0, $m9
	shl $m0, $m0, 2
	add $m6, $m6, $m0
	setzi $m0, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m0
	ld32 $m10, $mvertex_base, $m15, 3
	sub $m10, $m10, $m0
	cmpslt $m1, $m10, $mzero
	brz $m1, .Lnew_65_nonempty_\@
	setzi $m10, 0
.Lnew_65_nonempty_\@:
	// Capacity of this segment; m11 remains the worker stack pointer.
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	cmpult $m7, $m7, $m1
	brz $m7, .Lnew_65_capacity_\@
	setzi $m1, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
.Lnew_65_capacity_\@:
	cmpult $m7, $m1, $m10
	brz $m7, .Lnew_65_clamped_\@
	mov $m10, $m1
.Lnew_65_clamped_\@:
	mul $m0, $m0, $m5
	shl $m0, $m0, 1
	shl $m1, $m4, 5
	add $m0, $m0, $m1
	add $m2, $m2, $m0
	add $m3, $m3, $m0
	.endm

	.worker
	.p2align 3
.Lnew_65_split_max:
	INIT_new_65
.Lnew_65_split_max_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_65_done
	ROW_new_65
	ldconst $a6, 0xfbfffbff
	shr $m7, $m10, 4
	brz $m7, .Lnew_65_split_max_tail
	sub $m7, $m7, 1
.Lnew_65_split_max_panel:
	MAXP_new_65
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_65_split_max_panel
.Lnew_65_split_max_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_65_split_max_reduce
	and $m1, $m7, 1
	shr $m7, $m7, 1
	brz $m7, .Lnew_65_split_max_odd
	sub $m7, $m7, 1
.Lnew_65_split_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_65_split_max_pair
.Lnew_65_split_max_odd:
	brz $m1, .Lnew_65_split_max_reduce
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32tof16 $a0, $a0
	f16v2max $a6, $a6, $a0
.Lnew_65_split_max_reduce:
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_65_split_max_row

	.p2align 3
.Lnew_65_split_exp:
	INIT_new_65
.Lnew_65_split_exp_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_65_done
	ROW_new_65
	shl $m0, $m9, 2
	sub $m0, $m6, $m0
	ld32 $a0, $m0, $m15, 0
	ld32 $a1, $m0, $m15, 1
	ld32 $a2, $m0, $m15, 2
	f32max $a0, $a0, $a1
	f32max $a0, $a0, $a2
	f32tof16 $a4, $a0
	mov $a5, $a4
	brnz $m9, .Lnew_65_split_scale
	ldconst $a1, ATTENTION_SCALE_BITS
	f32mul $a2, $a0, $a1
	ld32 $m0, $mvertex_base, $m15, 0
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m1, $m5, $m1
	add $m0, $m0, $m1
	shl $m1, $m4, 2
	add $m0, $m0, $m1
	st32 $a2, $m0, $m15, 0
.Lnew_65_split_scale:
	ldconst $a0, ATTENTION_SCALE_BITS
	f32sub $a1, $azero, $a0
	f32v2tof16 $a0, $a0:1
	uput $TAS, $a0
	setzi $a6, 0
	setzi $a7, 0
	shr $m7, $m10, 4
	brz $m7, .Lnew_65_split_exp_tail
	sub $m7, $m7, 1
.Lnew_65_split_exp_panel:
	EXPP_new_65
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_65_split_exp_panel
.Lnew_65_split_exp_tail:
	and $m7, $m10, 15
	brz $m7, .Lnew_65_split_padding
	f16v2tof32 $a2:3, $a4
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a5, $a2, $a4
	f32sub $a5, $azero, $a5
	shr $m7, $m7, 1
	brz $m7, .Lnew_65_split_exp_odd
	sub $m7, $m7, 1
.Lnew_65_split_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_65_split_exp_pair
.Lnew_65_split_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_65_split_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_65_split_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_65_split_after_tail
	sub $m7, $m7, 1
.Lnew_65_split_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_65_split_zero_pair
.Lnew_65_split_after_tail:
	add $m2, $m2, $m8
.Lnew_65_split_padding:
	setzi $m7, ((ATTENTION_KEY_BLOCK_COLUMNS + 47) / 48) * 16
	mul $m0, $m9, $m7
	setzi $m1, ATTENTION_KEY_BLOCK_COLUMNS
	sub $m1, $m1, $m0
	cmpult $m0, $m7, $m1
	brz $m0, .Lnew_65_split_padding_capacity
	mov $m1, $m7
.Lnew_65_split_padding_capacity:
	shr $m7, $m1, 4
	add $m0, $m10, 15
	shr $m0, $m0, 4
	sub $m7, $m7, $m0
	brz $m7, .Lnew_65_split_store_sum
	sub $m7, $m7, 1
.Lnew_65_split_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_65_split_zero_panel
.Lnew_65_split_store_sum:
	setzi $m0, 12
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	f32add $a6, $a6, $a7
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 2
	bri .Lnew_65_split_exp_row

	.p2align 3
.Lnew_65_split_sum:
	ld32 $m5, $mvertex_base, $m15, 2
	ld32 $m2, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2 + 4
	mul $m0, $m5, $m0
	add $m2, $m2, $m0
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
.Lnew_65_split_sum_row:
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_65_done
	shl $m0, $m4, 2
	add $m3, $m2, $m0
	setzi $m0, 16
	mul $m0, $m5, $m0
	add $m6, $m2, $m0
	setzi $m0, 12
	mul $m0, $m4, $m0
	add $m6, $m6, $m0
	ld32 $a0, $m6, $m15, 0
	ld32 $a1, $m6, $m15, 1
	ld32 $a2, $m6, $m15, 2
	f32add $a0, $a0, $a1
	f32add $a0, $a0, $a2
	st32 $a0, $m3, $m15, 0
	add $m4, $m4, 6
	bri .Lnew_65_split_sum_row