#include "/srv/home/gc-sdk/ipu-stack/device/static_runtime.S"
#include "/srv/home/gc-sdk/ipu-stack/device/worker_support.S"

#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_0
#define REARRANGE_LOGICAL_ROWS 288
#define REARRANGE_LOGICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 288
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_0_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_0_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_0_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_0_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_0_rearrange_amp_left_f16_exit
#endif
.Lasm0_0_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_0_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_0_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_0_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_0_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_0_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_0_rearrange_amp_left_f16_next_row

.Lasm0_0_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_0_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_0_rearrange_amp_left_f16_zero_row_panel

.Lasm0_0_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_0_rearrange_amp_left_f16_row
.Lasm0_0_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_1
#define REARRANGE_LOGICAL_ROWS 73
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 73
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_1_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_1_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_1_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_1_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_1_rearrange_amp_left_f16_exit
#endif
.Lasm0_1_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_1_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_1_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_1_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_1_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_1_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_1_rearrange_amp_left_f16_next_row

.Lasm0_1_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_1_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_1_rearrange_amp_left_f16_zero_row_panel

.Lasm0_1_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_1_rearrange_amp_left_f16_row
.Lasm0_1_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_2
#define REARRANGE_LOGICAL_ROWS 53
#define REARRANGE_LOGICAL_COLUMNS 192
#define REARRANGE_PHYSICAL_COLUMNS 192
#define REARRANGE_PHYSICAL_ROWS 53
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_2_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_2_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_2_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_2_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_2_rearrange_amp_left_f16_exit
#endif
.Lasm0_2_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_2_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_2_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_2_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_2_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_2_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_2_rearrange_amp_left_f16_next_row

.Lasm0_2_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_2_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_2_rearrange_amp_left_f16_zero_row_panel

.Lasm0_2_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_2_rearrange_amp_left_f16_row
.Lasm0_2_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_3
#define REARRANGE_LOGICAL_ROWS 52
#define REARRANGE_LOGICAL_COLUMNS 192
#define REARRANGE_PHYSICAL_COLUMNS 192
#define REARRANGE_PHYSICAL_ROWS 52
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_3_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_3_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_3_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_3_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_3_rearrange_amp_left_f16_exit
#endif
.Lasm0_3_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_3_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_3_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_3_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_3_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_3_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_3_rearrange_amp_left_f16_next_row

.Lasm0_3_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_3_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_3_rearrange_amp_left_f16_zero_row_panel

.Lasm0_3_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_3_rearrange_amp_left_f16_row
.Lasm0_3_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_4
#define REARRANGE_LOGICAL_ROWS 61
#define REARRANGE_LOGICAL_COLUMNS 384
#define REARRANGE_PHYSICAL_COLUMNS 384
#define REARRANGE_PHYSICAL_ROWS 61
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_4_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_4_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_4_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_4_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_4_rearrange_amp_left_f16_exit
#endif
.Lasm0_4_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_4_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_4_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_4_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_4_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_4_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_4_rearrange_amp_left_f16_next_row

.Lasm0_4_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_4_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_4_rearrange_amp_left_f16_zero_row_panel

.Lasm0_4_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_4_rearrange_amp_left_f16_row
.Lasm0_4_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_5
#define REARRANGE_LOGICAL_ROWS 122
#define REARRANGE_LOGICAL_COLUMNS 288
#define REARRANGE_PHYSICAL_COLUMNS 288
#define REARRANGE_PHYSICAL_ROWS 122
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_5_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_5_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_5_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_5_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_5_rearrange_amp_left_f16_exit
#endif
.Lasm0_5_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_5_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_5_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_5_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_5_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_5_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_5_rearrange_amp_left_f16_next_row

.Lasm0_5_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_5_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_5_rearrange_amp_left_f16_zero_row_panel

.Lasm0_5_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_5_rearrange_amp_left_f16_row
.Lasm0_5_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_6
#define REARRANGE_LOGICAL_ROWS 81
#define REARRANGE_LOGICAL_COLUMNS 160
#define REARRANGE_PHYSICAL_COLUMNS 160
#define REARRANGE_PHYSICAL_ROWS 81
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_6_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_6_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_6_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_6_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_6_rearrange_amp_left_f16_exit
#endif
.Lasm0_6_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_6_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_6_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_6_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_6_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_6_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_6_rearrange_amp_left_f16_next_row

.Lasm0_6_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_6_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_6_rearrange_amp_left_f16_zero_row_panel

.Lasm0_6_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_6_rearrange_amp_left_f16_row
.Lasm0_6_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_7
#define REARRANGE_LOGICAL_ROWS 92
#define REARRANGE_LOGICAL_COLUMNS 36
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 92
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_7_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_7_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_7_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_7_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_7_rearrange_amp_left_f16_exit
#endif
.Lasm0_7_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_7_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_7_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_7_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_7_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_7_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_7_rearrange_amp_left_f16_next_row

.Lasm0_7_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_7_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_7_rearrange_amp_left_f16_zero_row_panel

.Lasm0_7_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_7_rearrange_amp_left_f16_row
.Lasm0_7_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_8
#define REARRANGE_LOGICAL_ROWS 83
#define REARRANGE_LOGICAL_COLUMNS 104
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 83
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_8_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_8_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_8_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_8_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_8_rearrange_amp_left_f16_exit
#endif
.Lasm0_8_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_8_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_8_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_8_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_8_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_8_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_8_rearrange_amp_left_f16_next_row

.Lasm0_8_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_8_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_8_rearrange_amp_left_f16_zero_row_panel

.Lasm0_8_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_8_rearrange_amp_left_f16_row
.Lasm0_8_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_9
#define REARRANGE_LOGICAL_ROWS 46
#define REARRANGE_LOGICAL_COLUMNS 116
#define REARRANGE_PHYSICAL_COLUMNS 128
#define REARRANGE_PHYSICAL_ROWS 46
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_9_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_9_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_9_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_9_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_9_rearrange_amp_left_f16_exit
#endif
.Lasm0_9_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_9_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_9_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_9_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_9_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_9_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_9_rearrange_amp_left_f16_next_row

.Lasm0_9_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_9_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_9_rearrange_amp_left_f16_zero_row_panel

.Lasm0_9_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_9_rearrange_amp_left_f16_row
.Lasm0_9_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_10
#define REARRANGE_LOGICAL_ROWS 31
#define REARRANGE_LOGICAL_COLUMNS 36
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 31
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_10_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_10_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_10_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_10_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_10_rearrange_amp_left_f16_exit
#endif
.Lasm0_10_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_10_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_10_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_10_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_10_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_10_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_10_rearrange_amp_left_f16_next_row

.Lasm0_10_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_10_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_10_rearrange_amp_left_f16_zero_row_panel

.Lasm0_10_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_10_rearrange_amp_left_f16_row
.Lasm0_10_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_11
#define REARRANGE_LOGICAL_ROWS 54
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_ROWS 54
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_11_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_11_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_11_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_11_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_11_rearrange_amp_left_f16_exit
#endif
.Lasm0_11_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_11_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_11_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_11_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_11_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_11_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_11_rearrange_amp_left_f16_next_row

.Lasm0_11_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_11_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_11_rearrange_amp_left_f16_zero_row_panel

.Lasm0_11_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_11_rearrange_amp_left_f16_row
.Lasm0_11_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_12
#define REARRANGE_LOGICAL_ROWS 60
#define REARRANGE_LOGICAL_COLUMNS 24
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 60
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_12_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_12_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_12_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_12_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_12_rearrange_amp_left_f16_exit
#endif
.Lasm0_12_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_12_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_12_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_12_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_12_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_12_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_12_rearrange_amp_left_f16_next_row

.Lasm0_12_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_12_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_12_rearrange_amp_left_f16_zero_row_panel

.Lasm0_12_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_12_rearrange_amp_left_f16_row
.Lasm0_12_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_13
#define REARRANGE_LOGICAL_ROWS 12
#define REARRANGE_LOGICAL_COLUMNS 50
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 12
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_13_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_13_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_13_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_13_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_13_rearrange_amp_left_f16_exit
#endif
.Lasm0_13_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_13_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_13_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_13_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_13_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_13_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_13_rearrange_amp_left_f16_next_row

.Lasm0_13_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_13_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_13_rearrange_amp_left_f16_zero_row_panel

.Lasm0_13_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_13_rearrange_amp_left_f16_row
.Lasm0_13_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_14
#define REARRANGE_LOGICAL_ROWS 33
#define REARRANGE_LOGICAL_COLUMNS 28
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 33
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_14_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_14_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_14_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_14_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_14_rearrange_amp_left_f16_exit
#endif
.Lasm0_14_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_14_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_14_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_14_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_14_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_14_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_14_rearrange_amp_left_f16_next_row

.Lasm0_14_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_14_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_14_rearrange_amp_left_f16_zero_row_panel

.Lasm0_14_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_14_rearrange_amp_left_f16_row
.Lasm0_14_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_15
#define REARRANGE_LOGICAL_ROWS 18
#define REARRANGE_LOGICAL_COLUMNS 62
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 18
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_15_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_15_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_15_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_15_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_15_rearrange_amp_left_f16_exit
#endif
.Lasm0_15_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_15_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_15_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_15_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_15_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_15_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_15_rearrange_amp_left_f16_next_row

.Lasm0_15_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_15_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_15_rearrange_amp_left_f16_zero_row_panel

.Lasm0_15_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_15_rearrange_amp_left_f16_row
.Lasm0_15_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_16
#define REARRANGE_LOGICAL_ROWS 34
#define REARRANGE_LOGICAL_COLUMNS 38
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 34
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_16_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_16_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_16_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_16_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_16_rearrange_amp_left_f16_exit
#endif
.Lasm0_16_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_16_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_16_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_16_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_16_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_16_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_16_rearrange_amp_left_f16_next_row

.Lasm0_16_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_16_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_16_rearrange_amp_left_f16_zero_row_panel

.Lasm0_16_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_16_rearrange_amp_left_f16_row
.Lasm0_16_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_17
#define REARRANGE_LOGICAL_ROWS 44
#define REARRANGE_LOGICAL_COLUMNS 108
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 44
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_17_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_17_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_17_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_17_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_17_rearrange_amp_left_f16_exit
#endif
.Lasm0_17_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_17_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_17_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_17_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_17_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_17_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_17_rearrange_amp_left_f16_next_row

.Lasm0_17_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_17_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_17_rearrange_amp_left_f16_zero_row_panel

.Lasm0_17_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_17_rearrange_amp_left_f16_row
.Lasm0_17_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_18
#define REARRANGE_LOGICAL_ROWS 12
#define REARRANGE_LOGICAL_COLUMNS 6
#define REARRANGE_PHYSICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_ROWS 12
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_18_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_18_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_18_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_18_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_18_rearrange_amp_left_f16_exit
#endif
.Lasm0_18_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_18_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_18_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_18_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_18_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_18_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_18_rearrange_amp_left_f16_next_row

.Lasm0_18_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_18_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_18_rearrange_amp_left_f16_zero_row_panel

.Lasm0_18_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_18_rearrange_amp_left_f16_row
.Lasm0_18_rearrange_amp_left_f16_exit:
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0_19
#define REARRANGE_LOGICAL_ROWS 17
#define REARRANGE_LOGICAL_COLUMNS 42
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 17
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_amp_left_f16
#endif
#ifndef REARRANGE_LOGICAL_ROWS
#define REARRANGE_LOGICAL_ROWS 1
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS REARRANGE_LOGICAL_ROWS
#endif
#ifndef REARRANGE_LOGICAL_COLUMNS
#define REARRANGE_LOGICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS REARRANGE_LOGICAL_COLUMNS
#endif
#if REARRANGE_LOGICAL_COLUMNS % 2 != 0
#error "AMP-left F16 packing requires an even logical column count"
#endif
#if REARRANGE_PHYSICAL_COLUMNS % 16 != 0
#error "AMP-left F16 packing requires complete physical panels"
#endif
#if REARRANGE_PHYSICAL_COLUMNS < ((REARRANGE_LOGICAL_COLUMNS + 15) / 16) * 16
#error "AMP-left F16 packing requires enough physical columns"
#endif

#define REARRANGE_FULL_PANELS (REARRANGE_LOGICAL_COLUMNS / 16)
#define REARRANGE_TAIL_COLUMNS (REARRANGE_LOGICAL_COLUMNS % 16)
#define REARRANGE_USED_PANELS ((REARRANGE_LOGICAL_COLUMNS + 15) / 16)
#define REARRANGE_ZERO_PANELS ((REARRANGE_PHYSICAL_COLUMNS / 16) - REARRANGE_USED_PANELS)

	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	add $m11, $m11, -16
	st32 $m9, $m11, $m15, 2
.Lasm0_19_rearrange_matrix:
	st32 $m3, $m11, $m15, 0
	st32 $m2, $m11, $m15, 1
	setzi $m0, .Lasm0_19_rearrange_amp_left_f16_worker
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, 1
	add $m3, $m3, $m0
	shl $m0, $m5, 5
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_19_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL

	// AMP-left storage consists of row-major 16-column panels. Assign rows to
	// workers so every panel is written by all six contexts in bank-friendly
	// order. Aligned rows move four F16 values per load/store pair.
	.worker
	.p2align 3
.Lasm0_19_rearrange_amp_left_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m4, $mvertex_base, $m15, 2
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 32
	mul $m4, $m4, $m5
	sub $m4, $m4, 32
	get $m6, $WSR
	and $m6, $m6, CSR_W_WSR__CTXTID_M1__MASK
#if REARRANGE_PHYSICAL_ROWS < 6
	// Inactive workers must not zero nonexistent rows past the allocation.
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brz $m2, .Lasm0_19_rearrange_amp_left_f16_exit
#endif
.Lasm0_19_rearrange_amp_left_f16_row:
	// Source rows are contiguous; destination rows advance through 16-column
	// panels with matrices * physicalRows * 32 bytes between panel starts.
	setzi $m2, REARRANGE_LOGICAL_COLUMNS
	mul $m9, $m6, $m2
	shl $m9, $m9, 1
	add $m9, $m0, $m9
	add $m10, $m6, 0
	shl $m10, $m10, 5
	add $m10, $m1, $m10

	setzi $m3, REARRANGE_LOGICAL_ROWS
	cmpult $m3, $m6, $m3
	brz $m3, .Lasm0_19_rearrange_amp_left_f16_zero_row

#if REARRANGE_FULL_PANELS > 0
	setzi $m7, REARRANGE_FULL_PANELS
.Lasm0_19_rearrange_amp_left_f16_full_panel:
#if REARRANGE_LOGICAL_COLUMNS % 4 == 0
	.rept 4
	ld64step $a0:1, $mzero, $m9+=, 1
	st64step $a0:1, $mzero, $m10+=, 1
	.endr
#else
	.rept 8
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
#endif
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_19_rearrange_amp_left_f16_full_panel
#endif

#if REARRANGE_TAIL_COLUMNS > 0
	.rept REARRANGE_TAIL_COLUMNS / 2
	ld32step $a0, $mzero, $m9+=, 1
	st32step $a0, $mzero, $m10+=, 1
	.endr
	zero $a0
	.rept (16 - REARRANGE_TAIL_COLUMNS) / 2
	st32step $a0, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
#endif

#if REARRANGE_ZERO_PANELS > 0
	setzi $m7, REARRANGE_ZERO_PANELS
.Lasm0_19_rearrange_amp_left_f16_zero_tail_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_19_rearrange_amp_left_f16_zero_tail_panel
#endif
	bri .Lasm0_19_rearrange_amp_left_f16_next_row

.Lasm0_19_rearrange_amp_left_f16_zero_row:
	setzi $m7, REARRANGE_PHYSICAL_COLUMNS / 16
.Lasm0_19_rearrange_amp_left_f16_zero_row_panel:
	.rept 4
	st64step $azeros, $mzero, $m10+=, 1
	.endr
	add $m10, $m10, $m4
	sub $m7, $m7, 1
	brnz $m7, .Lasm0_19_rearrange_amp_left_f16_zero_row_panel

.Lasm0_19_rearrange_amp_left_f16_next_row:
	add $m6, $m6, 6
	setzi $m2, REARRANGE_PHYSICAL_ROWS
	cmpult $m2, $m6, $m2
	brnz $m2, .Lasm0_19_rearrange_amp_left_f16_row
.Lasm0_19_rearrange_amp_left_f16_exit:
	exitz $m15

#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_0.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_0
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_0
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_1.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_1
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_1
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_2.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_2
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_2
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_3.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_3
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_3
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_4.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_4
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_4
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_5.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_5
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_5
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_6.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_6
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_6
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_7.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_7
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_7
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_8.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_8
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_8
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_9.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_9
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_9
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_10.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_10
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_10
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_11.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_11
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_11
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_12.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_12
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_12
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_13.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_13
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_13
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_14.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_14
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_14
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_15.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_15
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_15
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_16.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_16
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_16
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_17.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_17
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_17
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_18.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_18
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_18
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack1_19.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_19
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_19
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_0.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_0
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_0
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_1.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_1
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_1
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_2.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_2
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_2
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_3.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_3
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_3
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_4.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_4
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_4
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_5.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_5
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_5
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_6.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_6
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_6
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_7.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_7
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_7
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_8.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_8
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_8
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_9.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_9
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_9
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_10.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_10
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_10
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_11.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_11
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_11
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_12.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_12
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_12
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_13.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_13
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_13
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_14.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_14
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_14
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_15.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_15
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_15
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_16.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_16
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_16
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_17.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_17
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_17
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_18.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_18
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_18
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/copy-ldst-20260926/pack-audit/pack-fp16/pack2_19.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_19
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_19
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"

#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_0
#define REARRANGE_LOGICAL_ROWS 288
#define REARRANGE_LOGICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 288
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_0_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_0_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_0_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_0 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_0_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_0_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_0_rearrange_zero_panel
	brz $m8, .Lasm3_0_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_0 \offset, 1
	.endr
	bri .Lasm3_0_rearrange_next_row
.Lasm3_0_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_0 \offset, 0
	.endr
	bri .Lasm3_0_rearrange_next_row
.Lasm3_0_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_0_rearrange_next_row:
#else
	zero $m9
.Lasm3_0_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_0_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_0_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_0_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_0_rearrange_block_major_f16_transpose
.Lasm3_0_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_0_rearrange_block_major_f16_transpose
.Lasm3_0_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_0_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_0_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_0_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_1
#define REARRANGE_LOGICAL_ROWS 36
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_1_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_1_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_1_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_1 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_1_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_1_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_1_rearrange_zero_panel
	brz $m8, .Lasm3_1_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_1 \offset, 1
	.endr
	bri .Lasm3_1_rearrange_next_row
.Lasm3_1_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_1 \offset, 0
	.endr
	bri .Lasm3_1_rearrange_next_row
.Lasm3_1_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_1_rearrange_next_row:
#else
	zero $m9
.Lasm3_1_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_1_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_1_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_1_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_1_rearrange_block_major_f16_transpose
.Lasm3_1_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_1_rearrange_block_major_f16_transpose
.Lasm3_1_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_1_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_1_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_1_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_2
#define REARRANGE_LOGICAL_ROWS 24
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_2_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_2_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_2_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_2 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_2_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_2_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_2_rearrange_zero_panel
	brz $m8, .Lasm3_2_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_2 \offset, 1
	.endr
	bri .Lasm3_2_rearrange_next_row
.Lasm3_2_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_2 \offset, 0
	.endr
	bri .Lasm3_2_rearrange_next_row
.Lasm3_2_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_2_rearrange_next_row:
#else
	zero $m9
.Lasm3_2_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_2_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_2_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_2_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_2_rearrange_block_major_f16_transpose
.Lasm3_2_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_2_rearrange_block_major_f16_transpose
.Lasm3_2_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_2_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_2_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_2_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_3
#define REARRANGE_LOGICAL_ROWS 24
#define REARRANGE_LOGICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_3_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_3_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_3_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_3 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_3_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_3_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_3_rearrange_zero_panel
	brz $m8, .Lasm3_3_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_3 \offset, 1
	.endr
	bri .Lasm3_3_rearrange_next_row
.Lasm3_3_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_3 \offset, 0
	.endr
	bri .Lasm3_3_rearrange_next_row
.Lasm3_3_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_3_rearrange_next_row:
#else
	zero $m9
.Lasm3_3_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_3_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_3_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_3_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_3_rearrange_block_major_f16_transpose
.Lasm3_3_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_3_rearrange_block_major_f16_transpose
.Lasm3_3_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_3_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_3_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_3_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_4
#define REARRANGE_LOGICAL_ROWS 28
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_4_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_4_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_4_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_4 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_4_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_4_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_4_rearrange_zero_panel
	brz $m8, .Lasm3_4_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_4 \offset, 1
	.endr
	bri .Lasm3_4_rearrange_next_row
.Lasm3_4_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_4 \offset, 0
	.endr
	bri .Lasm3_4_rearrange_next_row
.Lasm3_4_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_4_rearrange_next_row:
#else
	zero $m9
.Lasm3_4_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_4_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_4_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_4_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_4_rearrange_block_major_f16_transpose
.Lasm3_4_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_4_rearrange_block_major_f16_transpose
.Lasm3_4_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_4_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_4_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_4_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_5
#define REARRANGE_LOGICAL_ROWS 60
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_5_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_5_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_5_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_5 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_5_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_5_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_5_rearrange_zero_panel
	brz $m8, .Lasm3_5_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_5 \offset, 1
	.endr
	bri .Lasm3_5_rearrange_next_row
.Lasm3_5_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_5 \offset, 0
	.endr
	bri .Lasm3_5_rearrange_next_row
.Lasm3_5_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_5_rearrange_next_row:
#else
	zero $m9
.Lasm3_5_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_5_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_5_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_5_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_5_rearrange_block_major_f16_transpose
.Lasm3_5_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_5_rearrange_block_major_f16_transpose
.Lasm3_5_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_5_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_5_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_5_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_6
#define REARRANGE_LOGICAL_ROWS 44
#define REARRANGE_LOGICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_6_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_6_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_6_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_6 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_6_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_6_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_6_rearrange_zero_panel
	brz $m8, .Lasm3_6_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_6 \offset, 1
	.endr
	bri .Lasm3_6_rearrange_next_row
.Lasm3_6_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_6 \offset, 0
	.endr
	bri .Lasm3_6_rearrange_next_row
.Lasm3_6_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_6_rearrange_next_row:
#else
	zero $m9
.Lasm3_6_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_6_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_6_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_6_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_6_rearrange_block_major_f16_transpose
.Lasm3_6_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_6_rearrange_block_major_f16_transpose
.Lasm3_6_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_6_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_6_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_6_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_7
#define REARRANGE_LOGICAL_ROWS 56
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_7_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_7_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_7_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_7 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_7_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_7_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_7_rearrange_zero_panel
	brz $m8, .Lasm3_7_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_7 \offset, 1
	.endr
	bri .Lasm3_7_rearrange_next_row
.Lasm3_7_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_7 \offset, 0
	.endr
	bri .Lasm3_7_rearrange_next_row
.Lasm3_7_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_7_rearrange_next_row:
#else
	zero $m9
.Lasm3_7_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_7_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_7_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_7_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_7_rearrange_block_major_f16_transpose
.Lasm3_7_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_7_rearrange_block_major_f16_transpose
.Lasm3_7_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_7_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_7_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_7_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_8
#define REARRANGE_LOGICAL_ROWS 48
#define REARRANGE_LOGICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_COLUMNS 128
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_8_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_8_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_8_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_8 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_8_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_8_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_8_rearrange_zero_panel
	brz $m8, .Lasm3_8_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_8 \offset, 1
	.endr
	bri .Lasm3_8_rearrange_next_row
.Lasm3_8_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_8 \offset, 0
	.endr
	bri .Lasm3_8_rearrange_next_row
.Lasm3_8_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_8_rearrange_next_row:
#else
	zero $m9
.Lasm3_8_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_8_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_8_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_8_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_8_rearrange_block_major_f16_transpose
.Lasm3_8_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_8_rearrange_block_major_f16_transpose
.Lasm3_8_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_8_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_8_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_8_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_9
#define REARRANGE_LOGICAL_ROWS 36
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_9_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_9_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_9_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_9 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_9_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_9_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_9_rearrange_zero_panel
	brz $m8, .Lasm3_9_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_9 \offset, 1
	.endr
	bri .Lasm3_9_rearrange_next_row
.Lasm3_9_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_9 \offset, 0
	.endr
	bri .Lasm3_9_rearrange_next_row
.Lasm3_9_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_9_rearrange_next_row:
#else
	zero $m9
.Lasm3_9_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_9_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_9_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_9_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_9_rearrange_block_major_f16_transpose
.Lasm3_9_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_9_rearrange_block_major_f16_transpose
.Lasm3_9_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_9_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_9_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_9_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_10
#define REARRANGE_LOGICAL_ROWS 32
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_10_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_10_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_10_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_10 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_10_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_10_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_10_rearrange_zero_panel
	brz $m8, .Lasm3_10_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_10 \offset, 1
	.endr
	bri .Lasm3_10_rearrange_next_row
.Lasm3_10_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_10 \offset, 0
	.endr
	bri .Lasm3_10_rearrange_next_row
.Lasm3_10_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_10_rearrange_next_row:
#else
	zero $m9
.Lasm3_10_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_10_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_10_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_10_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_10_rearrange_block_major_f16_transpose
.Lasm3_10_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_10_rearrange_block_major_f16_transpose
.Lasm3_10_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_10_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_10_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_10_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_11
#define REARRANGE_LOGICAL_ROWS 72
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_11_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_11_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_11_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_11 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_11_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_11_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_11_rearrange_zero_panel
	brz $m8, .Lasm3_11_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_11 \offset, 1
	.endr
	bri .Lasm3_11_rearrange_next_row
.Lasm3_11_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_11 \offset, 0
	.endr
	bri .Lasm3_11_rearrange_next_row
.Lasm3_11_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_11_rearrange_next_row:
#else
	zero $m9
.Lasm3_11_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_11_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_11_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_11_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_11_rearrange_block_major_f16_transpose
.Lasm3_11_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_11_rearrange_block_major_f16_transpose
.Lasm3_11_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_11_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_11_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_11_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_12
#define REARRANGE_LOGICAL_ROWS 92
#define REARRANGE_LOGICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_12_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_12_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_12_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_12 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_12_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_12_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_12_rearrange_zero_panel
	brz $m8, .Lasm3_12_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_12 \offset, 1
	.endr
	bri .Lasm3_12_rearrange_next_row
.Lasm3_12_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_12 \offset, 0
	.endr
	bri .Lasm3_12_rearrange_next_row
.Lasm3_12_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_12_rearrange_next_row:
#else
	zero $m9
.Lasm3_12_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_12_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_12_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_12_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_12_rearrange_block_major_f16_transpose
.Lasm3_12_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_12_rearrange_block_major_f16_transpose
.Lasm3_12_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_12_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_12_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_12_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_13
#define REARRANGE_LOGICAL_ROWS 84
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_13_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_13_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_13_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_13 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_13_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_13_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_13_rearrange_zero_panel
	brz $m8, .Lasm3_13_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_13 \offset, 1
	.endr
	bri .Lasm3_13_rearrange_next_row
.Lasm3_13_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_13 \offset, 0
	.endr
	bri .Lasm3_13_rearrange_next_row
.Lasm3_13_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_13_rearrange_next_row:
#else
	zero $m9
.Lasm3_13_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_13_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_13_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_13_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_13_rearrange_block_major_f16_transpose
.Lasm3_13_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_13_rearrange_block_major_f16_transpose
.Lasm3_13_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_13_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_13_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_13_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_14
#define REARRANGE_LOGICAL_ROWS 48
#define REARRANGE_LOGICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_COLUMNS 128
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_14_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_14_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_14_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_14 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_14_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_14_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_14_rearrange_zero_panel
	brz $m8, .Lasm3_14_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_14 \offset, 1
	.endr
	bri .Lasm3_14_rearrange_next_row
.Lasm3_14_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_14 \offset, 0
	.endr
	bri .Lasm3_14_rearrange_next_row
.Lasm3_14_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_14_rearrange_next_row:
#else
	zero $m9
.Lasm3_14_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_14_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_14_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_14_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_14_rearrange_block_major_f16_transpose
.Lasm3_14_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_14_rearrange_block_major_f16_transpose
.Lasm3_14_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_14_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_14_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_14_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_15
#define REARRANGE_LOGICAL_ROWS 16
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_15_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_15_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_15_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_15 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_15_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_15_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_15_rearrange_zero_panel
	brz $m8, .Lasm3_15_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_15 \offset, 1
	.endr
	bri .Lasm3_15_rearrange_next_row
.Lasm3_15_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_15 \offset, 0
	.endr
	bri .Lasm3_15_rearrange_next_row
.Lasm3_15_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_15_rearrange_next_row:
#else
	zero $m9
.Lasm3_15_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_15_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_15_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_15_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_15_rearrange_block_major_f16_transpose
.Lasm3_15_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_15_rearrange_block_major_f16_transpose
.Lasm3_15_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_15_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_15_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_15_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_16
#define REARRANGE_LOGICAL_ROWS 88
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_16_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_16_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_16_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_16 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_16_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_16_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_16_rearrange_zero_panel
	brz $m8, .Lasm3_16_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_16 \offset, 1
	.endr
	bri .Lasm3_16_rearrange_next_row
.Lasm3_16_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_16 \offset, 0
	.endr
	bri .Lasm3_16_rearrange_next_row
.Lasm3_16_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_16_rearrange_next_row:
#else
	zero $m9
.Lasm3_16_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_16_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_16_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_16_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_16_rearrange_block_major_f16_transpose
.Lasm3_16_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_16_rearrange_block_major_f16_transpose
.Lasm3_16_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_16_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_16_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_16_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_17
#define REARRANGE_LOGICAL_ROWS 76
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_17_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_17_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_17_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_17 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_17_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_17_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_17_rearrange_zero_panel
	brz $m8, .Lasm3_17_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_17 \offset, 1
	.endr
	bri .Lasm3_17_rearrange_next_row
.Lasm3_17_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_17 \offset, 0
	.endr
	bri .Lasm3_17_rearrange_next_row
.Lasm3_17_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_17_rearrange_next_row:
#else
	zero $m9
.Lasm3_17_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_17_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_17_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_17_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_17_rearrange_block_major_f16_transpose
.Lasm3_17_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_17_rearrange_block_major_f16_transpose
.Lasm3_17_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_17_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_17_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_17_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_18
#define REARRANGE_LOGICAL_ROWS 12
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_18_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_18_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_18_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_18 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_18_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_18_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_18_rearrange_zero_panel
	brz $m8, .Lasm3_18_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_18 \offset, 1
	.endr
	bri .Lasm3_18_rearrange_next_row
.Lasm3_18_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_18 \offset, 0
	.endr
	bri .Lasm3_18_rearrange_next_row
.Lasm3_18_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_18_rearrange_next_row:
#else
	zero $m9
.Lasm3_18_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_18_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_18_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_18_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_18_rearrange_block_major_f16_transpose
.Lasm3_18_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_18_rearrange_block_major_f16_transpose
.Lasm3_18_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_18_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_18_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_18_rearrange_block_major_f16_row_pair
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_19
#define REARRANGE_LOGICAL_ROWS 12
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#ifndef REARRANGE_CALL_SYMBOL
#define REARRANGE_CALL_SYMBOL rearrange_row_major_to_block_major_f16
#endif
#ifndef REARRANGE_PHYSICAL_COLUMNS
#define REARRANGE_PHYSICAL_COLUMNS 16
#endif
#ifndef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_PHYSICAL_ROWS 64
#endif

#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_19_rearrange_block_major_f16_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_19_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_19_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


	// Transpose each 2x2 F16 cell while converting a row-major RxN matrix
	// into Rx16-panel block-major coefficient order. Each 64-bit source load
	// covers four adjacent columns from one row. sort4x16lo/hi pair the same
	// column from two adjacent rows, which is the natural source order expected
	// by the GEMM supervisor's ld*putcs coefficient routing.

	// offset selects four columns (one 64-bit word) in each source row.
	.macro pack_four_columns_19 offset, second_row
	ld64 $a0:1, $m6, $m15, \offset
	.if \second_row
	ld64 $a2:3, $m7, $m15, \offset
	.else
	mov $a2:3, $azeros
	.endif
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, \offset * 32
	st32 $a5, $m10, $m15, \offset * 32 + 8
	st32 $a6, $m10, $m15, \offset * 32 + 16
	st32 $a7, $m10, $m15, \offset * 32 + 24
	.endm

	.worker
	.p2align 3
.Lasm3_19_rearrange_block_major_f16_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	ld32 $m3, $mvertex_base, $m15, 3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 1
.Lasm3_19_rearrange_block_major_f16_row_pair:
	// m6/m7 are the two source rows. They are only dereferenced when the
	// corresponding logical row exists.
	mul $m5, $m4, $m3
	shl $m5, $m5, 1
	add $m6, $m0, $m5
	shl $m5, $m3, 1
	add $m7, $m6, $m5
	add $m5, $m4, 1
	cmpult $m8, $m5, $m2

#if REARRANGE_LOGICAL_COLUMNS == 16 && REARRANGE_PHYSICAL_COLUMNS == 16
	// A complete 16-column panel needs no column indexing or per-word bounds
	// checks. Keep the uncommon final single row and zero panels out of its loop.
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m1, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_19_rearrange_zero_panel
	brz $m8, .Lasm3_19_rearrange_single_row
	.irp offset, 0, 1, 2, 3
	pack_four_columns_19 \offset, 1
	.endr
	bri .Lasm3_19_rearrange_next_row
.Lasm3_19_rearrange_single_row:
	.irp offset, 0, 1, 2, 3
	pack_four_columns_19 \offset, 0
	.endr
	bri .Lasm3_19_rearrange_next_row
.Lasm3_19_rearrange_zero_panel:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m10, $m15, \column * 8
	.endr
.Lasm3_19_rearrange_next_row:
#else
	zero $m9
.Lasm3_19_rearrange_block_major_f16_columns:
	// An Rx16 column block consists of R/16 coefficient panels.
	// Select the column block, the 16-row panel within it, and the row and
	// column offsets within that panel.
	shr $m5, $m9, 4
	setzi $m10, REARRANGE_PHYSICAL_ROWS * 32
	mul $m5, $m5, $m10
	add $m10, $m1, $m5
	shr $m5, $m4, 4
	shl $m5, $m5, 9
	add $m10, $m10, $m5
	and $m5, $m4, 15
	shl $m5, $m5, 1
	add $m10, $m10, $m5
	and $m5, $m9, 15
	shl $m5, $m5, 5
	add $m10, $m10, $m5

	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_19_rearrange_block_major_f16_zero_rows
	cmpult $m5, $m9, $m3
	brz $m5, .Lasm3_19_rearrange_block_major_f16_zero_rows
	ld64 $a0:1, $m6, $m15, 0
	brz $m8, .Lasm3_19_rearrange_block_major_f16_zero_second_row
	ld64 $a2:3, $m7, $m15, 0
	bri .Lasm3_19_rearrange_block_major_f16_transpose
.Lasm3_19_rearrange_block_major_f16_zero_second_row:
	mov $a2:3, $azeros
	bri .Lasm3_19_rearrange_block_major_f16_transpose
.Lasm3_19_rearrange_block_major_f16_zero_rows:
	mov $a0:1, $azeros
	mov $a2:3, $azeros
.Lasm3_19_rearrange_block_major_f16_transpose:
	sort4x16lo $a4, $a0, $a2
	sort4x16hi $a5, $a0, $a2
	sort4x16lo $a6, $a1, $a3
	sort4x16hi $a7, $a1, $a3
	st32 $a4, $m10, $m15, 0
	st32 $a5, $m10, $m15, 8
	st32 $a6, $m10, $m15, 16
	st32 $a7, $m10, $m15, 24
	add $m6, $m6, 8
	add $m7, $m7, 8
	add $m9, $m9, 4
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m9, $m5
	brnz $m5, .Lasm3_19_rearrange_block_major_f16_columns

#endif

	add $m4, $m4, 12
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_19_rearrange_block_major_f16_row_pair
	exitz $m15