#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lpack_f8_worker
#include "rearrange_call.inc"

// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lpack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lpack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lpack_f8_zero
.Lpack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lpack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lpack_f8_columns
	bri .Lpack_f8_next
.Lpack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lpack_f8_zero
.Lpack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lpack_f8_rows
	exitz $m15