#include "/srv/home/gc-sdk/ipu-stack/device/static_runtime.S"
#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_1
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_1_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_1_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_1_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_1_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_1_max_tail
	sub $m7, $m7, 1
#endif
.Lold_1_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_1_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_1_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_1_max_scalar
	sub $m7, $m7, 1
.Lold_1_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_1_max_pair
.Lold_1_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_1_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_1_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_1_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_1
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_1_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_1
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_1
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_1_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_1_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_1_exp_odd
	sub $m7, $m7, 1
.Lold_1_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_1_exp_pair
.Lold_1_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_1_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_1_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_1_zero_panels_setup
	sub $m7, $m7, 1
.Lold_1_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_1_zero_pair
.Lold_1_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_1_sum_ready
	sub $m7, $m7, 1
.Lold_1_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_1_zero_panel
.Lold_1_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_1_row
.Lold_1_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_1
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_1_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_1_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_1_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_1_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_1_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_1_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_1_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_1_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_1_max_scalar
	sub $m7, $m7, 1
.Lnew_1_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_1_max_pair
.Lnew_1_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_1_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_1_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_1_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_1
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_1_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_1
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_1
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_1_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_1_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_1_exp_odd
	sub $m7, $m7, 1
.Lnew_1_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_1
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_1_exp_pair
.Lnew_1_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_1_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_1_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_1_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_1_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_1_zero_pair
.Lnew_1_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_1_sum_ready
	sub $m7, $m7, 1
.Lnew_1_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_1_zero_panel
.Lnew_1_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_1_row
.Lnew_1_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_2
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_2_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_2_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_2_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_2_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_2_max_tail
	sub $m7, $m7, 1
#endif
.Lold_2_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_2_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_2_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_2_max_scalar
	sub $m7, $m7, 1
.Lold_2_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_2_max_pair
.Lold_2_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_2_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_2_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_2_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_2
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_2_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_2
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_2
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_2_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_2_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_2_exp_odd
	sub $m7, $m7, 1
.Lold_2_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_2_exp_pair
.Lold_2_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_2_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_2_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_2_zero_panels_setup
	sub $m7, $m7, 1
.Lold_2_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_2_zero_pair
.Lold_2_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_2_sum_ready
	sub $m7, $m7, 1
.Lold_2_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_2_zero_panel
.Lold_2_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_2_row
.Lold_2_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_2
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_2_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_2_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_2_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_2_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_2_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_2_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_2_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_2_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_2_max_scalar
	sub $m7, $m7, 1
.Lnew_2_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_2_max_pair
.Lnew_2_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_2_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_2_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_2_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_2
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_2_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_2
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_2
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_2_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_2_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_2_exp_odd
	sub $m7, $m7, 1
.Lnew_2_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_2
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_2_exp_pair
.Lnew_2_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_2_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_2_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_2_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_2_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_2_zero_pair
.Lnew_2_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_2_sum_ready
	sub $m7, $m7, 1
.Lnew_2_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_2_zero_panel
.Lnew_2_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_2_row
.Lnew_2_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_15
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_15_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_15_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_15_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_15_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_15_max_tail
	sub $m7, $m7, 1
#endif
.Lold_15_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_15_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_15_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_15_max_scalar
	sub $m7, $m7, 1
.Lold_15_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_15_max_pair
.Lold_15_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_15_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_15_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_15_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_15
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_15_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_15
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_15
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_15_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_15_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_15_exp_odd
	sub $m7, $m7, 1
.Lold_15_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_15_exp_pair
.Lold_15_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_15_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_15_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_15_zero_panels_setup
	sub $m7, $m7, 1
.Lold_15_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_15_zero_pair
.Lold_15_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_15_sum_ready
	sub $m7, $m7, 1
.Lold_15_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_15_zero_panel
.Lold_15_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_15_row
.Lold_15_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_15
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_15_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_15_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_15_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_15_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_15_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_15_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_15_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_15_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_15_max_scalar
	sub $m7, $m7, 1
.Lnew_15_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_15_max_pair
.Lnew_15_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_15_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_15_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_15_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_15
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_15_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_15
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_15
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_15_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_15_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_15_exp_odd
	sub $m7, $m7, 1
.Lnew_15_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_15
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_15_exp_pair
.Lnew_15_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_15_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_15_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_15_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_15_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_15_zero_pair
.Lnew_15_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_15_sum_ready
	sub $m7, $m7, 1
.Lnew_15_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_15_zero_panel
.Lnew_15_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_15_row
.Lnew_15_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_16
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_16_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_16_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_16_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_16_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_16_max_tail
	sub $m7, $m7, 1
#endif
.Lold_16_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_16_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_16_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_16_max_scalar
	sub $m7, $m7, 1
.Lold_16_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_16_max_pair
.Lold_16_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_16_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_16_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_16_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_16
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_16_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_16
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_16
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_16_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_16_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_16_exp_odd
	sub $m7, $m7, 1
.Lold_16_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_16_exp_pair
.Lold_16_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_16_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_16_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_16_zero_panels_setup
	sub $m7, $m7, 1
.Lold_16_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_16_zero_pair
.Lold_16_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_16_sum_ready
	sub $m7, $m7, 1
.Lold_16_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_16_zero_panel
.Lold_16_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_16_row
.Lold_16_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 16
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_16
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_16_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_16_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_16_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_16_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_16_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_16_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_16_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_16_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_16_max_scalar
	sub $m7, $m7, 1
.Lnew_16_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_16_max_pair
.Lnew_16_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_16_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_16_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_16_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_16
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_16_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_16
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_16
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_16_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_16_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_16_exp_odd
	sub $m7, $m7, 1
.Lnew_16_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_16
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_16_exp_pair
.Lnew_16_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_16_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_16_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_16_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_16_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_16_zero_pair
.Lnew_16_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_16_sum_ready
	sub $m7, $m7, 1
.Lnew_16_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_16_zero_panel
.Lnew_16_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_16_row
.Lnew_16_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_17
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_17_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_17_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_17_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_17_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_17_max_tail
	sub $m7, $m7, 1
#endif
.Lold_17_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_17_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_17_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_17_max_scalar
	sub $m7, $m7, 1
.Lold_17_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_17_max_pair
.Lold_17_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_17_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_17_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_17_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_17
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_17_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_17
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_17
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_17_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_17_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_17_exp_odd
	sub $m7, $m7, 1
.Lold_17_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_17_exp_pair
.Lold_17_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_17_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_17_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_17_zero_panels_setup
	sub $m7, $m7, 1
.Lold_17_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_17_zero_pair
.Lold_17_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_17_sum_ready
	sub $m7, $m7, 1
.Lold_17_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_17_zero_panel
.Lold_17_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_17_row
.Lold_17_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_17
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_17_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_17_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_17_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_17_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_17_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_17_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_17_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_17_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_17_max_scalar
	sub $m7, $m7, 1
.Lnew_17_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_17_max_pair
.Lnew_17_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_17_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_17_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_17_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_17
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_17_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_17
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_17
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_17_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_17_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_17_exp_odd
	sub $m7, $m7, 1
.Lnew_17_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_17
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_17_exp_pair
.Lnew_17_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_17_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_17_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_17_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_17_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_17_zero_pair
.Lnew_17_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_17_sum_ready
	sub $m7, $m7, 1
.Lnew_17_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_17_zero_panel
.Lnew_17_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_17_row
.Lnew_17_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_31
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_31_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_31_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_31_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_31_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_31_max_tail
	sub $m7, $m7, 1
#endif
.Lold_31_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_31_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_31_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_31_max_scalar
	sub $m7, $m7, 1
.Lold_31_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_31_max_pair
.Lold_31_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_31_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_31_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_31_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_31
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_31_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_31
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_31
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_31_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_31_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_31_exp_odd
	sub $m7, $m7, 1
.Lold_31_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_31_exp_pair
.Lold_31_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_31_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_31_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_31_zero_panels_setup
	sub $m7, $m7, 1
.Lold_31_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_31_zero_pair
.Lold_31_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_31_sum_ready
	sub $m7, $m7, 1
.Lold_31_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_31_zero_panel
.Lold_31_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_31_row
.Lold_31_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 32
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_31
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_31_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_31_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_31_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_31_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_31_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_31_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_31_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_31_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_31_max_scalar
	sub $m7, $m7, 1
.Lnew_31_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_31_max_pair
.Lnew_31_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_31_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_31_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_31_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_31
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_31_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_31
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_31
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_31_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_31_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_31_exp_odd
	sub $m7, $m7, 1
.Lnew_31_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_31
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_31_exp_pair
.Lnew_31_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_31_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_31_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_31_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_31_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_31_zero_pair
.Lnew_31_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_31_sum_ready
	sub $m7, $m7, 1
.Lnew_31_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_31_zero_panel
.Lnew_31_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_31_row
.Lnew_31_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 64
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_64
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_64_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_64_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_64_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_64_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_64_max_tail
	sub $m7, $m7, 1
#endif
.Lold_64_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_64_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_64_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_64_max_scalar
	sub $m7, $m7, 1
.Lold_64_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_64_max_pair
.Lold_64_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_64_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_64_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_64_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_64
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_64_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_64
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_64
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_64_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_64_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_64_exp_odd
	sub $m7, $m7, 1
.Lold_64_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_64_exp_pair
.Lold_64_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_64_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_64_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_64_zero_panels_setup
	sub $m7, $m7, 1
.Lold_64_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_64_zero_pair
.Lold_64_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_64_sum_ready
	sub $m7, $m7, 1
.Lold_64_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_64_zero_panel
.Lold_64_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_64_row
.Lold_64_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 64
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_64
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_64_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_64_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_64_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_64_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_64_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_64_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_64_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_64_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_64_max_scalar
	sub $m7, $m7, 1
.Lnew_64_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_64_max_pair
.Lnew_64_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_64_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_64_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_64_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_64
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_64_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_64
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_64
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_64_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_64_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_64_exp_odd
	sub $m7, $m7, 1
.Lnew_64_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_64
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_64_exp_pair
.Lnew_64_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_64_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_64_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_64_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_64_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_64_zero_pair
.Lnew_64_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_64_sum_ready
	sub $m7, $m7, 1
.Lnew_64_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_64_zero_panel
.Lnew_64_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_64_row
.Lnew_64_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 80
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_65
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_65_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_65_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_65_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_65_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_65_max_tail
	sub $m7, $m7, 1
#endif
.Lold_65_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_65_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_65_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_65_max_scalar
	sub $m7, $m7, 1
.Lold_65_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_65_max_pair
.Lold_65_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_65_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_65_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_65_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_65
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_65_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_65
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_65
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_65_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_65_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_65_exp_odd
	sub $m7, $m7, 1
.Lold_65_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_65_exp_pair
.Lold_65_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_65_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_65_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_65_zero_panels_setup
	sub $m7, $m7, 1
.Lold_65_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_65_zero_pair
.Lold_65_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_65_sum_ready
	sub $m7, $m7, 1
.Lold_65_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_65_zero_panel
.Lold_65_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_65_row
.Lold_65_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 80
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_65
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_65_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_65_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_65_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_65_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_65_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_65_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_65_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_65_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_65_max_scalar
	sub $m7, $m7, 1
.Lnew_65_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_65_max_pair
.Lnew_65_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_65_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_65_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_65_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_65
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_65_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_65
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_65
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_65_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_65_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_65_exp_odd
	sub $m7, $m7, 1
.Lnew_65_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_65
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_65_exp_pair
.Lnew_65_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_65_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_65_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_65_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_65_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_65_zero_pair
.Lnew_65_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_65_sum_ready
	sub $m7, $m7, 1
.Lnew_65_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_65_zero_panel
.Lnew_65_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_65_row
.Lnew_65_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 736
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_729
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_729_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_729_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_729_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_729_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_729_max_tail
	sub $m7, $m7, 1
#endif
.Lold_729_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_729_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_729_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_729_max_scalar
	sub $m7, $m7, 1
.Lold_729_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_729_max_pair
.Lold_729_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_729_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_729_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_729_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_729
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_729_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_729
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_729
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_729_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_729_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_729_exp_odd
	sub $m7, $m7, 1
.Lold_729_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_729
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_729_exp_pair
.Lold_729_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_729_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_729_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_729_zero_panels_setup
	sub $m7, $m7, 1
.Lold_729_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_729_zero_pair
.Lold_729_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_729_sum_ready
	sub $m7, $m7, 1
.Lold_729_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_729_zero_panel
.Lold_729_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_729_row
.Lold_729_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 0
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 736
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_729
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_729_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_729_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_729_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_729_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_729_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_729_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_729_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_729_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_729_max_scalar
	sub $m7, $m7, 1
.Lnew_729_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_729_max_pair
.Lnew_729_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_729_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_729_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_729_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_729
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_729_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_729
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_729
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_729_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_729_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_729_exp_odd
	sub $m7, $m7, 1
.Lnew_729_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_729
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_729_exp_pair
.Lnew_729_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_729_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_729_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_729_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_729_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_729_zero_pair
.Lnew_729_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_729_sum_ready
	sub $m7, $m7, 1
.Lnew_729_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_729_zero_panel
.Lnew_729_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_729_row
.Lnew_729_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 768
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_old_768
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lold_768_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lold_768_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lold_768_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lold_768_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_768_max_tail
	sub $m7, $m7, 1
#endif
.Lold_768_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lold_768_max_panel
#if !ATTENTION_FULL_BLOCK
.Lold_768_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_768_max_scalar
	sub $m7, $m7, 1
.Lold_768_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lold_768_max_pair
.Lold_768_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lold_768_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lold_768_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lold_768_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_old_768
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lold_768_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_old_768
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_old_768
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_768_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lold_768_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lold_768_exp_odd
	sub $m7, $m7, 1
.Lold_768_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_old_768
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lold_768_exp_pair
.Lold_768_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lold_768_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lold_768_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lold_768_zero_panels_setup
	sub $m7, $m7, 1
.Lold_768_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lold_768_zero_pair
.Lold_768_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lold_768_sum_ready
	sub $m7, $m7, 1
.Lold_768_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lold_768_zero_panel
.Lold_768_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lold_768_row
.Lold_768_done:
	exitz $m15

#undef ATTENTION_FULL_BLOCK
#define ATTENTION_FULL_BLOCK 1
#undef ATTENTION_KEY_BLOCK_COLUMNS
#define ATTENTION_KEY_BLOCK_COLUMNS 768
#undef ATTENTION_SCALE_BITS
#define ATTENTION_SCALE_BITS 0x3df15bf0
#undef ATTENTION_SOFTMAX_SYMBOL
#define ATTENTION_SOFTMAX_SYMBOL softmax_new_768
#undef SOFTMAX_FRAME_BYTES
#include "arch/gc_tile_defines.h"

// AMP-left panel order; FP16 probabilities with FP32 maxima and sums.
// m2 weights, m3 scores, m4 query rows, m5 valid keys.
#if ATTENTION_FULL_BLOCK
#define SOFTMAX_FRAME_BYTES 12
#else
#define SOFTMAX_FRAME_BYTES 16
#endif
	.section .text.ATTENTION_SOFTMAX_SYMBOL,"ax",@progbits
	.globl ATTENTION_SOFTMAX_SYMBOL
	.p2align 2
	.type ATTENTION_SOFTMAX_SYMBOL,@function
ATTENTION_SOFTMAX_SYMBOL:
	.supervisor
	add $m11, $m11, -SOFTMAX_FRAME_BYTES
	st32 $m2, $m11, $m15, 0
	st32 $m3, $m11, $m15, 1
	st32 $m4, $m11, $m15, 2
#if !ATTENTION_FULL_BLOCK
	st32 $m5, $m11, $m15, 3
#endif
	setzi $m0, .Lnew_768_worker
	runall $m0, $m11, 0
	sync TEXCH_SYNCZONE_LOCAL
	add $m11, $m11, SOFTMAX_FRAME_BYTES
	br $m10
	.size ATTENTION_SOFTMAX_SYMBOL, .-ATTENTION_SOFTMAX_SYMBOL

	.worker
	.p2align 3
.Lnew_768_worker:
	ld32 $m5, $mvertex_base, $m15, 2
#if !ATTENTION_FULL_BLOCK
	ld32 $m10, $mvertex_base, $m15, 3
#endif
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	cmpult $m0, $m4, $m5
	brz $m0, .Lnew_768_done
	add $m8, $m5, -1
	shl $m8, $m8, 5
.Lnew_768_row:
	ld32 $m2, $mvertex_base, $m15, 0
	ld32 $m3, $mvertex_base, $m15, 1
	shl $m0, $m4, 5
	add $m3, $m3, $m0
	add $m2, $m2, $m0
	mov $m9, $m3
	// The maximum of finite FP16 inputs is exactly representable in FP16.
	ldconst $a6, 0xfbfffbff
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_768_max_tail
	sub $m7, $m7, 1
#endif
.Lnew_768_max_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	{ ld32step $a0, $mzero, $m3+=, 1
	  f16v2max $a6, $a6, $a0 }
	.endr
	f16v2max $a6, $a6, $a0
	add $m3, $m3, $m8
	brnzdec $m7, .Lnew_768_max_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_768_max_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_768_max_scalar
	sub $m7, $m7, 1
.Lnew_768_max_pair:
	ld32step $a0, $mzero, $m3+=, 1
	f16v2max $a6, $a6, $a0
	brnzdec $m7, .Lnew_768_max_pair
.Lnew_768_max_scalar:
#endif
	f16v2tof32 $a6:7, $a6
	f32max $a6, $a6, $a7
#if !ATTENTION_FULL_BLOCK
	and $m0, $m10, 1
	brz $m0, .Lnew_768_max_ready
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32max $a6, $a6, $a0
.Lnew_768_max_ready:
#endif
	ldconst $a4, ATTENTION_SCALE_BITS
	f32mul $a7, $a6, $a4
	// Persistent maxima/denominators follow the packed weight matrix.
	ld32 $m6, $mvertex_base, $m15, 0
	setzi $m0, ATTENTION_KEY_BLOCK_COLUMNS * 2
	mul $m0, $m5, $m0
	add $m6, $m6, $m0
	shl $m0, $m4, 2
	add $m6, $m6, $m0
	st32 $a7, $m6, $m15, 0
	f32sub $a5, $azero, $a7
	setzi $a6, 0
	setzi $a7, 0
	mov $m3, $m9
#if ATTENTION_FULL_BLOCK
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
#else
	shr $m7, $m10, 4
	brz $m7, .Lnew_768_exp_tail
	sub $m7, $m7, 1
#endif
	.macro EXP_new_768
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	.endm
.Lnew_768_exp_panel:
	ld32step $a0, $mzero, $m3+=, 1
	.rept 7
	EXP_new_768
	{ ld32step $a0, $mzero, $m3+=, 1
	  f32v2add $a6:7, $a6:7, $a2:3 }
	.endr
	EXP_new_768
	f32v2add $a6:7, $a6:7, $a2:3
	add $m3, $m3, $m8
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_768_exp_panel
#if !ATTENTION_FULL_BLOCK
.Lnew_768_exp_tail:
	and $m7, $m10, 15
	shr $m7, $m7, 1
	brz $m7, .Lnew_768_exp_odd
	sub $m7, $m7, 1
.Lnew_768_exp_pair:
	ld32step $a0, $mzero, $m3+=, 1
	EXP_new_768
	f32v2add $a6:7, $a6:7, $a2:3
	brnzdec $m7, .Lnew_768_exp_pair
.Lnew_768_exp_odd:
	and $m0, $m10, 1
	brz $m0, .Lnew_768_zero_tail
	ld32 $a0, $m3, $m15, 0
	f16v2tof32 $a0:1, $a0
	f32v2mul $a0:1, $a4:B, $a0:1
	f32v2add $a0:1, $a5:B, $a0:1
	// Mask the unused lane before exponentiation; it contributes exactly zero.
	ldconst $a1, 0xc77fe000
	f32v2tof16 $a0, $a0:1
	f16v2exp $a0, $a0
	{ st32step $a0, $mzero, $m2+=, 1
	  f16v2tof32 $a2:3, $a0 }
	f32v2add $a6:7, $a6:7, $a2:3
.Lnew_768_zero_tail:
	and $m7, $m10, 15
	add $m7, $m7, 1
	shr $m7, $m7, 1
	setzi $m0, 8
	sub $m7, $m0, $m7
	brz $m7, .Lnew_768_zero_panels_setup
	sub $m7, $m7, 1
.Lnew_768_zero_pair:
	st32step $mzero, $mzero, $m2+=, 1
	brnzdec $m7, .Lnew_768_zero_pair
.Lnew_768_zero_panels_setup:
	add $m2, $m2, $m8
	shr $m0, $m10, 4
	setzi $m7, ATTENTION_KEY_BLOCK_COLUMNS / 16 - 1
	sub $m7, $m7, $m0
	brz $m7, .Lnew_768_sum_ready
	sub $m7, $m7, 1
.Lnew_768_zero_panel:
	.rept 8
	st32step $mzero, $mzero, $m2+=, 1
	.endr
	add $m2, $m2, $m8
	brnzdec $m7, .Lnew_768_zero_panel
.Lnew_768_sum_ready:
#endif
	f32add $a6, $a6, $a7
	shl $m0, $m5, 2
	add $m6, $m6, $m0
	st32 $a6, $m6, $m15, 0
	add $m4, $m4, 6
	cmpult $m0, $m4, $m5
	brnz $m0, .Lnew_768_row
.Lnew_768_done:
	exitz $m15