#include "/srv/home/gc-sdk/ipu-stack/device/static_runtime.S"
#include "/srv/home/gc-sdk/ipu-stack/device/worker_support.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/fp8-pack-check/pack0.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0
#define WORKER_CODELET_SYMBOL __runCodelet_Pack0
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/fp8-pack-check/pack1.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1
#define WORKER_CODELET_SYMBOL __runCodelet_Pack1
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/fp8-pack-check/pack2.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2
#define WORKER_CODELET_SYMBOL __runCodelet_Pack2
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/fp8-pack-check/pack3.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack3
#define WORKER_CODELET_SYMBOL __runCodelet_Pack3
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"

#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm0
#define REARRANGE_LOGICAL_COLUMNS 240
#define REARRANGE_PHYSICAL_COLUMNS 240
#define REARRANGE_PHYSICAL_ROWS 288
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm0_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm0_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm0_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm0_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm0_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm0_pack_f8_zero
.Lasm0_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm0_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm0_pack_f8_columns
	bri .Lasm0_pack_f8_next
.Lasm0_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm0_pack_f8_zero
.Lasm0_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm0_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm1
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm1_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm1_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm1_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm1_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm1_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm1_pack_f8_zero
.Lasm1_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm1_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm1_pack_f8_columns
	bri .Lasm1_pack_f8_next
.Lasm1_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm1_pack_f8_zero
.Lasm1_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm1_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm2
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm2_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm2_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm2_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm2_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm2_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm2_pack_f8_zero
.Lasm2_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm2_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm2_pack_f8_columns
	bri .Lasm2_pack_f8_next
.Lasm2_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm2_pack_f8_zero
.Lasm2_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm2_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_pack_f8_zero
.Lasm3_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_pack_f8_columns
	bri .Lasm3_pack_f8_next
.Lasm3_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_pack_f8_zero
.Lasm3_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm4
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm4_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm4_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm4_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm4_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm4_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm4_pack_f8_zero
.Lasm4_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm4_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm4_pack_f8_columns
	bri .Lasm4_pack_f8_next
.Lasm4_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm4_pack_f8_zero
.Lasm4_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm4_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm5
#define REARRANGE_LOGICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm5_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm5_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm5_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm5_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm5_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm5_pack_f8_zero
.Lasm5_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm5_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm5_pack_f8_columns
	bri .Lasm5_pack_f8_next
.Lasm5_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm5_pack_f8_zero
.Lasm5_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm5_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm6
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm6_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm6_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm6_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm6_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm6_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm6_pack_f8_zero
.Lasm6_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm6_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm6_pack_f8_columns
	bri .Lasm6_pack_f8_next
.Lasm6_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm6_pack_f8_zero
.Lasm6_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm6_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm7
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm7_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm7_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm7_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm7_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm7_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm7_pack_f8_zero
.Lasm7_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm7_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm7_pack_f8_columns
	bri .Lasm7_pack_f8_next
.Lasm7_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm7_pack_f8_zero
.Lasm7_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm7_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm8
#define REARRANGE_LOGICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm8_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm8_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm8_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm8_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm8_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm8_pack_f8_zero
.Lasm8_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm8_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm8_pack_f8_columns
	bri .Lasm8_pack_f8_next
.Lasm8_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm8_pack_f8_zero
.Lasm8_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm8_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm9
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm9_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm9_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm9_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm9_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm9_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm9_pack_f8_zero
.Lasm9_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm9_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm9_pack_f8_columns
	bri .Lasm9_pack_f8_next
.Lasm9_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm9_pack_f8_zero
.Lasm9_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm9_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm10
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm10_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm10_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm10_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm10_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm10_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm10_pack_f8_zero
.Lasm10_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm10_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm10_pack_f8_columns
	bri .Lasm10_pack_f8_next
.Lasm10_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm10_pack_f8_zero
.Lasm10_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm10_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm11
#define REARRANGE_LOGICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm11_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm11_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm11_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm11_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm11_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm11_pack_f8_zero
.Lasm11_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm11_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm11_pack_f8_columns
	bri .Lasm11_pack_f8_next
.Lasm11_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm11_pack_f8_zero
.Lasm11_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm11_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm12
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm12_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm12_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm12_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm12_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm12_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm12_pack_f8_zero
.Lasm12_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm12_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm12_pack_f8_columns
	bri .Lasm12_pack_f8_next
.Lasm12_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm12_pack_f8_zero
.Lasm12_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm12_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm13
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm13_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm13_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm13_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm13_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm13_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm13_pack_f8_zero
.Lasm13_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm13_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm13_pack_f8_columns
	bri .Lasm13_pack_f8_next
.Lasm13_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm13_pack_f8_zero
.Lasm13_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm13_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm14
#define REARRANGE_LOGICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_COLUMNS 128
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm14_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm14_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm14_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm14_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm14_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm14_pack_f8_zero
.Lasm14_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm14_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm14_pack_f8_columns
	bri .Lasm14_pack_f8_next
.Lasm14_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm14_pack_f8_zero
.Lasm14_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm14_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm15
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm15_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm15_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm15_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm15_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm15_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm15_pack_f8_zero
.Lasm15_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm15_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm15_pack_f8_columns
	bri .Lasm15_pack_f8_next
.Lasm15_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm15_pack_f8_zero
.Lasm15_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm15_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm16
#define REARRANGE_LOGICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm16_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm16_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm16_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm16_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm16_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm16_pack_f8_zero
.Lasm16_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm16_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm16_pack_f8_columns
	bri .Lasm16_pack_f8_next
.Lasm16_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm16_pack_f8_zero
.Lasm16_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm16_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm17
#define REARRANGE_LOGICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm17_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm17_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm17_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm17_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm17_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm17_pack_f8_zero
.Lasm17_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm17_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm17_pack_f8_columns
	bri .Lasm17_pack_f8_next
.Lasm17_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm17_pack_f8_zero
.Lasm17_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm17_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm18
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm18_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm18_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm18_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm18_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm18_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm18_pack_f8_zero
.Lasm18_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm18_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm18_pack_f8_columns
	bri .Lasm18_pack_f8_next
.Lasm18_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm18_pack_f8_zero
.Lasm18_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm18_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm19
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm19_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm19_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm19_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm19_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm19_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm19_pack_f8_zero
.Lasm19_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm19_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm19_pack_f8_columns
	bri .Lasm19_pack_f8_next
.Lasm19_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm19_pack_f8_zero
.Lasm19_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm19_pack_f8_rows
	exitz $m15