#include "/srv/home/gc-sdk/ipu-stack/device/static_runtime.S"
#include "/srv/home/gc-sdk/ipu-stack/device/worker_support.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_0.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_0
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_0
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_1.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_1
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_1
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_2.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_2
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_2
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_3.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_3
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_3
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_4.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_4
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_4
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_5.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_5
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_5
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_6.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_6
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_6
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_7.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_7
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_7
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_8.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_8
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_8
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_9.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_9
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_9
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_10.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_10
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_10
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_11.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_11
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_11
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_12.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_12
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_12
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_13.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_13
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_13
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_14.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_14
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_14
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_15.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_15
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_15
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_16.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_16
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_16
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_17.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_17
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_17
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_18.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_18
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_18
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack0_19.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack0_19
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack0_19
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_0.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_0
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_0
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_1.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_1
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_1
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_2.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_2
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_2
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_3.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_3
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_3
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_4.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_4
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_4
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_5.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_5
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_5
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_6.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_6
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_6
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_7.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_7
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_7
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_8.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_8
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_8
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_9.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_9
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_9
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_10.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_10
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_10
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_11.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_11
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_11
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_12.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_12
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_12
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_13.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_13
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_13
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_14.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_14
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_14
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_15.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_15
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_15
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_16.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_16
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_16
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_17.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_17
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_17
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_18.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_18
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_18
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack1_19.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack1_19
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack1_19
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_0.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_0
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_0
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_1.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_1
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_1
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_2.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_2
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_2
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_3.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_3
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_3
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_4.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_4
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_4
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_5.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_5
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_5
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_6.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_6
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_6
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_7.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_7
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_7
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_8.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_8
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_8
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_9.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_9
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_9
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_10.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_10
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_10
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_11.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_11
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_11
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_12.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_12
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_12
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_13.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_13
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_13
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_14.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_14
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_14
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_15.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_15
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_15
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_16.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_16
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_16
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_17.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_17
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_17
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_18.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_18
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_18
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"
#include "/srv/home/gc-sdk/ipu-stack/artifacts/kernel-cost-pack-final-20260922/pack/pack2_19.S"
#undef WORKER_CALL_SYMBOL
#undef WORKER_CODELET_SYMBOL
#undef WORKER_ARGUMENTS
#define WORKER_CALL_SYMBOL pack2_19
#define WORKER_CODELET_SYMBOL __runCodelet_Vertex_pack2_19
#define WORKER_ARGUMENTS $m3,$m2,$m4,$m5,$m6,$m7,$m8,$m9
#define WORKER_FRAME_BYTES 32
#include "/srv/home/gc-sdk/ipu-stack/device/worker_call.S"

#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_0
#define REARRANGE_LOGICAL_ROWS 288
#define REARRANGE_LOGICAL_COLUMNS 240
#define REARRANGE_PHYSICAL_COLUMNS 240
#define REARRANGE_PHYSICAL_ROWS 288
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_0_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_0_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_0_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_0_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_0_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_0_pack_f8_zero
.Lasm3_0_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_0_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_0_pack_f8_columns
	bri .Lasm3_0_pack_f8_next
.Lasm3_0_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_0_pack_f8_zero
.Lasm3_0_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_0_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_1
#define REARRANGE_LOGICAL_ROWS 28
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_1_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_1_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_1_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_1_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_1_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_1_pack_f8_zero
.Lasm3_1_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_1_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_1_pack_f8_columns
	bri .Lasm3_1_pack_f8_next
.Lasm3_1_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_1_pack_f8_zero
.Lasm3_1_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_1_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_2
#define REARRANGE_LOGICAL_ROWS 80
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_2_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_2_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_2_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_2_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_2_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_2_pack_f8_zero
.Lasm3_2_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_2_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_2_pack_f8_columns
	bri .Lasm3_2_pack_f8_next
.Lasm3_2_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_2_pack_f8_zero
.Lasm3_2_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_2_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_3
#define REARRANGE_LOGICAL_ROWS 60
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_3_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_3_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_3_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_3_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_3_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_3_pack_f8_zero
.Lasm3_3_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_3_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_3_pack_f8_columns
	bri .Lasm3_3_pack_f8_next
.Lasm3_3_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_3_pack_f8_zero
.Lasm3_3_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_3_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_4
#define REARRANGE_LOGICAL_ROWS 40
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_4_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_4_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_4_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_4_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_4_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_4_pack_f8_zero
.Lasm3_4_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_4_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_4_pack_f8_columns
	bri .Lasm3_4_pack_f8_next
.Lasm3_4_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_4_pack_f8_zero
.Lasm3_4_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_4_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_5
#define REARRANGE_LOGICAL_ROWS 88
#define REARRANGE_LOGICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_COLUMNS 128
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_5_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_5_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_5_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_5_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_5_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_5_pack_f8_zero
.Lasm3_5_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_5_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_5_pack_f8_columns
	bri .Lasm3_5_pack_f8_next
.Lasm3_5_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_5_pack_f8_zero
.Lasm3_5_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_5_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_6
#define REARRANGE_LOGICAL_ROWS 32
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_6_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_6_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_6_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_6_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_6_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_6_pack_f8_zero
.Lasm3_6_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_6_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_6_pack_f8_columns
	bri .Lasm3_6_pack_f8_next
.Lasm3_6_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_6_pack_f8_zero
.Lasm3_6_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_6_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_7
#define REARRANGE_LOGICAL_ROWS 20
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_7_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_7_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_7_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_7_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_7_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_7_pack_f8_zero
.Lasm3_7_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_7_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_7_pack_f8_columns
	bri .Lasm3_7_pack_f8_next
.Lasm3_7_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_7_pack_f8_zero
.Lasm3_7_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_7_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_8
#define REARRANGE_LOGICAL_ROWS 8
#define REARRANGE_LOGICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_8_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_8_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_8_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_8_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_8_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_8_pack_f8_zero
.Lasm3_8_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_8_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_8_pack_f8_columns
	bri .Lasm3_8_pack_f8_next
.Lasm3_8_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_8_pack_f8_zero
.Lasm3_8_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_8_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_9
#define REARRANGE_LOGICAL_ROWS 88
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_9_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_9_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_9_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_9_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_9_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_9_pack_f8_zero
.Lasm3_9_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_9_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_9_pack_f8_columns
	bri .Lasm3_9_pack_f8_next
.Lasm3_9_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_9_pack_f8_zero
.Lasm3_9_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_9_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_10
#define REARRANGE_LOGICAL_ROWS 32
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_10_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_10_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_10_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_10_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_10_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_10_pack_f8_zero
.Lasm3_10_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_10_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_10_pack_f8_columns
	bri .Lasm3_10_pack_f8_next
.Lasm3_10_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_10_pack_f8_zero
.Lasm3_10_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_10_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_11
#define REARRANGE_LOGICAL_ROWS 80
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 96
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_11_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_11_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_11_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_11_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_11_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_11_pack_f8_zero
.Lasm3_11_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_11_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_11_pack_f8_columns
	bri .Lasm3_11_pack_f8_next
.Lasm3_11_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_11_pack_f8_zero
.Lasm3_11_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_11_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_12
#define REARRANGE_LOGICAL_ROWS 52
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_12_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_12_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_12_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_12_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_12_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_12_pack_f8_zero
.Lasm3_12_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_12_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_12_pack_f8_columns
	bri .Lasm3_12_pack_f8_next
.Lasm3_12_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_12_pack_f8_zero
.Lasm3_12_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_12_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_13
#define REARRANGE_LOGICAL_ROWS 8
#define REARRANGE_LOGICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_13_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_13_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_13_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_13_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_13_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_13_pack_f8_zero
.Lasm3_13_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_13_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_13_pack_f8_columns
	bri .Lasm3_13_pack_f8_next
.Lasm3_13_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_13_pack_f8_zero
.Lasm3_13_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_13_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_14
#define REARRANGE_LOGICAL_ROWS 40
#define REARRANGE_LOGICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_14_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_14_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_14_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_14_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_14_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_14_pack_f8_zero
.Lasm3_14_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_14_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_14_pack_f8_columns
	bri .Lasm3_14_pack_f8_next
.Lasm3_14_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_14_pack_f8_zero
.Lasm3_14_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_14_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_15
#define REARRANGE_LOGICAL_ROWS 60
#define REARRANGE_LOGICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_COLUMNS 96
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_15_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_15_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_15_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_15_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_15_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_15_pack_f8_zero
.Lasm3_15_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_15_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_15_pack_f8_columns
	bri .Lasm3_15_pack_f8_next
.Lasm3_15_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_15_pack_f8_zero
.Lasm3_15_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_15_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_16
#define REARRANGE_LOGICAL_ROWS 16
#define REARRANGE_LOGICAL_COLUMNS 64
#define REARRANGE_PHYSICAL_COLUMNS 80
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_16_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_16_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_16_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_16_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_16_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_16_pack_f8_zero
.Lasm3_16_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_16_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_16_pack_f8_columns
	bri .Lasm3_16_pack_f8_next
.Lasm3_16_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_16_pack_f8_zero
.Lasm3_16_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_16_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_17
#define REARRANGE_LOGICAL_ROWS 4
#define REARRANGE_LOGICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_COLUMNS 48
#define REARRANGE_PHYSICAL_ROWS 32
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_17_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_17_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_17_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_17_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_17_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_17_pack_f8_zero
.Lasm3_17_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_17_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_17_pack_f8_columns
	bri .Lasm3_17_pack_f8_next
.Lasm3_17_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_17_pack_f8_zero
.Lasm3_17_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_17_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_18
#define REARRANGE_LOGICAL_ROWS 56
#define REARRANGE_LOGICAL_COLUMNS 16
#define REARRANGE_PHYSICAL_COLUMNS 32
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_18_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_18_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_18_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_18_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_18_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_18_pack_f8_zero
.Lasm3_18_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_18_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_18_pack_f8_columns
	bri .Lasm3_18_pack_f8_next
.Lasm3_18_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_18_pack_f8_zero
.Lasm3_18_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_18_pack_f8_rows
	exitz $m15


#undef REARRANGE_CALL_SYMBOL
#undef REARRANGE_WORKER
#undef REARRANGE_LOGICAL_ROWS
#undef REARRANGE_LOGICAL_COLUMNS
#undef REARRANGE_PHYSICAL_COLUMNS
#undef REARRANGE_PHYSICAL_ROWS
#define REARRANGE_CALL_SYMBOL asm3_19
#define REARRANGE_LOGICAL_ROWS 60
#define REARRANGE_LOGICAL_COLUMNS 112
#define REARRANGE_PHYSICAL_COLUMNS 128
#define REARRANGE_PHYSICAL_ROWS 64
#include "arch/gc_tile_defines.h"

#define REARRANGE_ELEMENT_SHIFT 0
#define REARRANGE_ARGUMENTS $m3,$m2,$m4,$m7
#define REARRANGE_WORKER .Lasm3_19_pack_f8_worker
// Shared synchronous matrix loop for dense packing workers.
// REARRANGE_ARGUMENTS follows the worker frame order.
#ifndef REARRANGE_ELEMENT_SHIFT
#define REARRANGE_ELEMENT_SHIFT 1
#endif
	.text
	.allow_optimizations
	.set SYNC_COMPUTE_SET, TEXCH_SYNCZONE_LOCAL

	.section .text.REARRANGE_CALL_SYMBOL,"ax",@progbits
	.globl REARRANGE_CALL_SYMBOL
	.p2align 2
	.type REARRANGE_CALL_SYMBOL,@function
REARRANGE_CALL_SYMBOL:
	.supervisor
	// Kernel ABI: m2 destination, m3 source, m4 logical rows,
	// m5 physical rows, m6 target order, m7 logical columns,
	// m8 physical columns, m9 matrices, m10 return address.
	add $m11, $m11, -16
.Lasm3_19_rearrange_matrix:
	.set rearrange_argument_index, 0
	.irp argument, REARRANGE_ARGUMENTS
	st32 \argument, $m11, $m15, rearrange_argument_index
	.set rearrange_argument_index, rearrange_argument_index + 1
	.endr
	setzi $m0, REARRANGE_WORKER
	runall $m0, $m11, 0
	sync SYNC_COMPUTE_SET
	mul $m0, $m4, $m7
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m3, $m3, $m0
	mul $m0, $m5, $m8
	shl $m0, $m0, REARRANGE_ELEMENT_SHIFT
	add $m2, $m2, $m0
	sub $m9, $m9, 1
	brnz $m9, .Lasm3_19_rearrange_matrix
	add $m11, $m11, 16
	br $m10
	.size REARRANGE_CALL_SYMBOL, .-REARRANGE_CALL_SYMBOL


// Four rows per worker iteration, sixteen columns per panel. Logical extents
// are multiples of four rows and sixteen columns; absent panels are zeroed.
// Source words may be in standard memory; no wide-load alignment is required.
	.worker
	.p2align 3
.Lasm3_19_pack_f8_worker:
	ld32 $m0, $mvertex_base, $m15, 0
	ld32 $m1, $mvertex_base, $m15, 1
	ld32 $m2, $mvertex_base, $m15, 2
	setzi $m3, REARRANGE_LOGICAL_COLUMNS
	shl $m9, $m3, 1
	add $m10, $m9, $m3
	get $m4, $WSR
	and $m4, $m4, CSR_W_WSR__CTXTID_M1__MASK
	shl $m4, $m4, 2
.Lasm3_19_pack_f8_rows:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	mul $m5, $m4, $m5
	add $m6, $m0, $m5
	shr $m5, $m4, 5
	shl $m5, $m5, 9
	add $m8, $m1, $m5
	and $m5, $m4, 31
	add $m8, $m8, $m5
	zero $m7
	cmpult $m5, $m4, $m2
	brz $m5, .Lasm3_19_pack_f8_zero
.Lasm3_19_pack_f8_columns:
	setzi $m5, REARRANGE_LOGICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brz $m5, .Lasm3_19_pack_f8_zero
	.irp word, 0, 1, 2, 3
	ld32 $a0, $m6, $m15, \word
	ld32 $a1, $m6, $m3, \word
	ld32 $a2, $m6, $m9, \word
	ld32 $a3, $m6, $m10, \word
	shuf8x8lo $a4, $a0, $a1
	shuf8x8hi $a5, $a0, $a1
	shuf8x8lo $a6, $a2, $a3
	shuf8x8hi $a7, $a2, $a3
	sort4x16lo $a0, $a4, $a6
	sort4x16hi $a1, $a4, $a6
	sort4x16lo $a2, $a5, $a7
	sort4x16hi $a3, $a5, $a7
	st32 $a0, $m8, $m15, \word * 32
	st32 $a1, $m8, $m15, \word * 32 + 8
	st32 $a2, $m8, $m15, \word * 32 + 16
	st32 $a3, $m8, $m15, \word * 32 + 24
	.endr
	add $m6, $m6, 16
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_19_pack_f8_columns
	bri .Lasm3_19_pack_f8_next
.Lasm3_19_pack_f8_zero:
	.irp column, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	st32 $m15, $m8, $m15, \column * 8
	.endr
	setzi $m5, REARRANGE_PHYSICAL_ROWS * 16
	add $m8, $m8, $m5
	add $m7, $m7, 16
	setzi $m5, REARRANGE_PHYSICAL_COLUMNS
	cmpult $m5, $m7, $m5
	brnz $m5, .Lasm3_19_pack_f8_zero
.Lasm3_19_pack_f8_next:
	add $m4, $m4, 24
	setzi $m5, REARRANGE_PHYSICAL_ROWS
	cmpult $m5, $m4, $m5
	brnz $m5, .Lasm3_19_pack_f8_rows
	exitz $m15