exchangeReplay phase=10 transfers=1 activeTiles=2 overlappingTiles=1 eventCycles=91 touchedSpans=3
2026-09-11T15:39:16.886293Z  INFO writing application package tiles=1472
2026-09-11T15:39:16.887304Z  INFO application package written
2026-09-11T15:39:16.887479Z  INFO waiting for hardware ownership path=artifacts/layout-sweep/device.lock
2026-09-11T15:39:16.887490Z  INFO acquired hardware ownership path=artifacts/layout-sweep/device.lock
2026-09-11T15:39:16.887499Z  INFO opening IPU device path="/dev/ipu0"
2026-09-11T15:39:16.887537Z  INFO IPU device attached path=/dev/ipu0
2026-09-11T15:39:16.887543Z  INFO initializing IPU device
2026-09-11T15:39:16.895455Z  INFO IPU device initialized
2026-09-11T15:39:16.895520Z  INFO replaying device configuration register_writes=2003
2026-09-11T15:39:16.899373Z  INFO loading application tile_count=1472 final_mark=23
2026-09-11T15:39:17.078786Z  INFO application loaded and initial synchronization completed tile_count=1472
2026-09-11T15:39:17.080267Z  INFO starting host exchange session startup_mark=23
2026-09-11T15:39:17.080286Z  INFO attaching host exchange pages pages=2
2026-09-11T15:39:17.081085Z  INFO host exchange session started
2026-09-11T15:39:17.081092Z  INFO invoking streaming host exchange call call="run" command=0 phases=3
2026-09-11T15:39:17.143372Z ERROR exchange replay failed error=device timeout: host call run phase 2/3: invalid device data: exchange replay phase 10 corrupted logical tile 1: [(622592, 152183779, 2006066147, Some([(1, 413696)]), Some([(1, 622592)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622596, 152175587, 2006057955, Some([(1, 413700)]), Some([(1, 622596)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622600, 152200163, 2006082531, Some([(1, 413704)]), Some([(1, 622600)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622604, 152191971, 2006074339, Some([(1, 413708)]), Some([(1, 622604)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622608, 152216547, 2006098915, Some([(1, 413712)]), Some([(1, 622608)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622612, 152208355, 2006090723, Some([(1, 413716)]), Some([(1, 622612)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622616, 152232931, 2006115299, Some([(1, 413720)]), Some([(1, 622616)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622620, 152224739, 2006107107, Some([(1, 413724)]), Some([(1, 622620)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622624, 152249315, 2006131683, Some([(1, 413728)]), Some([(1, 622624)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622628, 152241123, 2006123491, Some([(1, 413732)]), Some([(1, 622628)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622632, 152265699, 2006148067, Some([(1, 413736)]), Some([(1, 622632)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622636, 152257507, 2006139875, Some([(1, 413740)]), Some([(1, 622636)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622640, 152282083, 2006164451, Some([(1, 413744)]), Some([(1, 622640)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622644, 152273891, 2006156259, Some([(1, 413748)]), Some([(1, 622644)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622648, 152298467, 2006180835, Some([(1, 413752)]), Some([(1, 622648)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622652, 152290275, 2006172643, Some([(1, 413756)]), Some([(1, 622652)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], [])] device=counts=[0, 0, 1472, 0] firstUnexpected=[(0, 2), (1, 2), (2, 2), (3, 2), (4, 2), (5, 2), (6, 2), (7, 2), (8, 2), (9, 2), (10, 2), (11, 2), (12, 2), (13, 2), (14, 2), (15, 2)] contexts=[(0, 2, Ok(377056), Some("generated tile program+0x40"), Some((376992, 92, 56, [423624705, 413216875, 1098911744, 421527553, 422938388])), Some((23, 0, [])), None, Ok([311752, 327680, 393216, 393216, 1, 0, 360456, 377024, 1, 377008, 377056, 362208, 0, 0, 0, 0]), Ok((1, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (1, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (2, 2, Ok(377052), Some("generated tile program+0x3c"), Some((376992, 84, 52, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((21, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 377020, 1, 377004, 377052, 362208, 0, 0, 0, 0]), Ok((1, 31)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (3, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (4, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (5, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (6, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (7, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (8, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (9, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (10, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (11, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (12, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (13, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (14, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), []), (15, 2, Ok(377044), Some("generated tile program+0x34"), Some((376992, 76, 44, [423624705, 413216875, 1098911744, 421527553, 422952992])), Some((19, 0, [])), None, Ok([311752, 4, 393216, 393216, 1, 327680, 0, 4294967295, 0, 377004, 377044, 362208, 0, 0, 0, 0]), Ok((0, 23)), Ok(None), Ok([0, 0, 0, 0, 0, 0]), [])]
Error: device timeout: host call run phase 2/3: invalid device data: exchange replay phase 10 corrupted logical tile 1: [(622592, 152183779, 2006066147, Some([(1, 413696)]), Some([(1, 622592)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622596, 152175587, 2006057955, Some([(1, 413700)]), Some([(1, 622596)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622600, 152200163, 2006082531, Some([(1, 413704)]), Some([(1, 622600)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622604, 152191971, 2006074339, Some([(1, 413708)]), Some([(1, 622604)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622608, 152216547, 2006098915, Some([(1, 413712)]), Some([(1, 622608)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622612, 152208355, 2006090723, Some([(1, 413716)]), Some([(1, 622612)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622616, 152232931, 2006115299, Some([(1, 413720)]), Some([(1, 622616)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622620, 152224739, 2006107107, Some([(1, 413724)]), Some([(1, 622620)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622624, 152249315, 2006131683, Some([(1, 413728)]), Some([(1, 622624)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622628, 152241123, 2006123491, Some([(1, 413732)]), Some([(1, 622628)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622632, 152265699, 2006148067, Some([(1, 413736)]), Some([(1, 622632)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622636, 152257507, 2006139875, Some([(1, 413740)]), Some([(1, 622636)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622640, 152282083, 2006164451, Some([(1, 413744)]), Some([(1, 622640)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622644, 152273891, 2006156259, Some([(1, 413748)]), Some([(1, 622644)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622648, 152298467, 2006180835, Some([(1, 413752)]), Some([(1, 622648)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], []), (622652, 152290275, 2006172643, Some([(1, 413756)]), Some([(1, 622652)]), [(0, Receive, 54, 86, 91)], [(0, Send, 28, 60, 413696, 64), (0, Receive, 54, 86, 622592, 64)], [(0, 0, PartnerBusy, 28, 60, 413696, 64), (1, 0, Send, 28, 60, 413696, 64), (1, 0, Receive, 54, 86, 622592, 64)], [])]