namespace Aeshnidae.Mule; /// /// The console test harness. STAGING SCAFFOLDING: every verb refuses unless /// Settings.AllowTestHarness is true, which it is not by default and must never be on /// live - these commands create and destroy real items. /// /// Why it exists, and what it does and does not prove /// -------------------------------------------------- /// A vault has to be tested with hundreds of items, with the server killed mid-flight, and /// with two movements racing on one account. None of that can be done by hand in a client, /// and the parts that matter are not client-side: they are the ordering of the three steps, /// the single commit, the account lock and the accounting afterwards. /// /// So the harness drives the SAME core - Movement.MoveIntoVault, the same account lock, the /// same single confirmed write, the same ledger - with a STAND-IN PACK in place of a /// player's inventory. A stand-in pack is a real Container holding real WorldObjects with /// real biotas, so every database fact the real path produces, this produces. What it does /// NOT exercise is the Player half: the refusals that need a live session (trade window, /// wielded, too busy) and the client messages. Those are listed as unverified. /// /// It also keeps its own durable record of every item it created /// (`aeshnidae_mule_test_item`), which is what makes the crash test an accounting exercise /// rather than an opinion: after a SIGKILL and a restart, every object id it ever minted /// must be in exactly one place. /// public static class MuleTest { public const string PackTable = "aeshnidae_mule_test"; public const string ItemTable = "aeshnidae_mule_test_item"; /// Impenetrability VI - an ordinary item enchantment, for the fidelity test. private const uint EnchantSpellId = 1486; private static string _cs = ""; public static void Initialize() { var cfg = ConfigManager.Config?.MySql?.Shard; if (cfg is null) return; _cs = new MySqlConnectionStringBuilder { Server = cfg.Host, Port = cfg.Port, Database = cfg.Database, UserID = cfg.Username, Password = cfg.Password, }.ConnectionString; } private static MySqlConnection Open() { var conn = new MySqlConnection(_cs); conn.Open(); return conn; } private static void EnsureTables() { using var conn = Open(); using (var cmd = conn.CreateCommand()) { cmd.CommandText = $@" CREATE TABLE IF NOT EXISTS `{PackTable}` ( `account_Id` INT UNSIGNED NOT NULL, `container_Id` INT UNSIGNED NOT NULL, PRIMARY KEY (`account_Id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; cmd.ExecuteNonQuery(); } using (var cmd = conn.CreateCommand()) { cmd.CommandText = $@" CREATE TABLE IF NOT EXISTS `{ItemTable}` ( `account_Id` INT UNSIGNED NOT NULL, `object_Id` INT UNSIGNED NOT NULL, `wcid` INT UNSIGNED NOT NULL DEFAULT 0, `name` VARCHAR(128) NOT NULL DEFAULT '', PRIMARY KEY (`object_Id`), KEY `k_account` (`account_Id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; cmd.ExecuteNonQuery(); } } // ---- the stand-in pack --------------------------------------------------- private static readonly ConcurrentDictionary _packs = new(); /// /// The account's stand-in pack, loaded or minted. Same shape as a vault link and saved /// the same way, so it survives a kill and can be counted afterwards. /// public static Container? Pack(uint accountId, out string error) { error = ""; EnsureTables(); if (_packs.TryGetValue(accountId, out var cached)) return cached; uint guid = 0; using (var conn = Open()) using (var cmd = conn.CreateCommand()) { cmd.CommandText = $"SELECT `container_Id` FROM `{PackTable}` WHERE `account_Id` = @a;"; cmd.Parameters.AddWithValue("@a", accountId); var r = cmd.ExecuteScalar(); if (r is not (null or DBNull)) guid = Convert.ToUInt32(r); } ShardBiota? dbBiota = guid == 0 ? null : DatabaseManager.Shard.BaseDatabase.GetBiota(guid); if (guid != 0) { if (dbBiota is null) { // Harness scaffolding, not a vault: if something removed it - the // orphaned-biota purge test does exactly that, since a stand-in pack is // deliberately NOT protected - forget the row and mint a new one. A real // vault link in this state is refused instead and needs a human, because // items may be pointing at it (Vault.LoadNow). ModManager.Log($"[{Mod.Name}] mule-test: the stand-in pack 0x{guid:X8} has no biota row; forgetting it and minting a new one.", ModManager.LogLevel.Warn); using (var conn = Open()) using (var del = conn.CreateCommand()) { del.CommandText = $"DELETE FROM `{PackTable}` WHERE `account_Id` = @a;"; del.Parameters.AddWithValue("@a", accountId); del.ExecuteNonQuery(); } guid = 0; } } if (guid != 0 && dbBiota is not null) { if (WorldObjectFactory.CreateWorldObject(dbBiota) is not Container loaded) { error = $"0x{guid:X8} is not a container"; return null; } var deadline = DateTime.UtcNow.AddSeconds(Mod.Settings.LoadTimeoutSeconds); while (!loaded.InventoryLoaded) { if (DateTime.UtcNow > deadline) { error = "the stand-in pack did not finish loading"; return null; } Thread.Sleep(25); } _packs[accountId] = loaded; return loaded; } var weenie = DatabaseManager.World.GetCachedWeenie(Mod.Settings.VaultWcid); if (weenie is null) { error = $"weenie {Mod.Settings.VaultWcid} is not in the world database"; return null; } var fresh = WorldObjectFactory.CreateWorldObject(weenie, GuidManager.NewDynamicGuid()) as Container; if (fresh is null) { error = "the stand-in pack could not be created"; return null; } fresh.ItemCapacity = (byte)Mod.Settings.SlotsPerContainer; fresh.ContainerCapacity = 0; if (!SaveNow(fresh)) { error = "the stand-in pack could not be saved"; return null; } using (var conn = Open()) using (var cmd = conn.CreateCommand()) { cmd.CommandText = $"INSERT INTO `{PackTable}` (`account_Id`, `container_Id`) VALUES (@a, @c) ON DUPLICATE KEY UPDATE `container_Id` = @c;"; cmd.Parameters.AddWithValue("@a", accountId); cmd.Parameters.AddWithValue("@c", fresh.Guid.Full); cmd.ExecuteNonQuery(); } _packs[accountId] = fresh; return fresh; } /// A save that is waited on, so a test never races the database. public static bool SaveNow(WorldObject wo) { var ok = false; var done = new ManualResetEventSlim(false); wo.SaveBiotaToDatabase(false); DatabaseManager.Shard.SaveBiota(wo.Biota, wo.BiotaDatabaseLock, r => { ok = r; done.Set(); }); return done.Wait(TimeSpan.FromSeconds(30)) && ok; } private static void Record(uint accountId, WorldObject item) { using var conn = Open(); using var cmd = conn.CreateCommand(); cmd.CommandText = $"INSERT IGNORE INTO `{ItemTable}` (`account_Id`, `object_Id`, `wcid`, `name`) VALUES (@a, @o, @w, @n);"; cmd.Parameters.AddWithValue("@a", accountId); cmd.Parameters.AddWithValue("@o", item.Guid.Full); cmd.Parameters.AddWithValue("@w", item.WeenieClassId); cmd.Parameters.AddWithValue("@n", (item.Name ?? "").Length > 128 ? item.Name![..128] : item.Name ?? ""); cmd.ExecuteNonQuery(); } // ---- filling ------------------------------------------------------------- /// /// Mints real items into the stand-in pack and records every /// object id. Each is saved and confirmed before it counts as created, so the durable /// record and the database never disagree about what exists. /// public static string Fill(uint accountId, int count, uint wcid, int treasureProfile) { var pack = Pack(accountId, out var error); if (pack is null) return $"no stand-in pack: {error}"; var made = 0; var failed = 0; for (var i = 0; i < count; i++) { if (pack.Inventory.Count >= Mod.Settings.SlotsPerContainer) return $"the stand-in pack is full at {pack.Inventory.Count} - deposit what is in it first. {made} made."; WorldObject? item = null; if (treasureProfile > 0) { var profile = DatabaseManager.World.GetCachedDeathTreasure((uint)treasureProfile); if (profile is null) return $"treasure profile {treasureProfile} is not in the world database"; item = LootGenerationFactory.CreateRandomLootObjects_Test(profile, true); } else { item = WorldObjectFactory.CreateNewWorldObject(wcid); } if (item is null) { failed++; continue; } // Stamp things a round trip must not lose, so the before/after comparison has // something to compare: our own 9042/9043, workmanship, tinks and item XP. item.SetProperty((PropertyInt)9043, 7); item.SetProperty((PropertyInt)9042, 1); if (item.ItemWorkmanship is null) item.SetProperty(PropertyInt.ItemWorkmanship, 7); item.SetProperty(PropertyInt.NumTimesTinkered, 3); item.SetProperty(PropertyInt.ImbuedEffect, (int)ImbuedEffectType.CriticalStrike); item.SetProperty(PropertyInt64.ItemTotalXp, 1_234_567L); item.SetProperty(PropertyString.Inscription, $"mule test {accountId} #{i}"); // A live enchantment ON THE ITEM, in its own enchantment registry - the one // thing a vault cannot keep ticking (no landblock means no heartbeat), so the // round trip has to prove at least that the row itself survives untouched. try { var spell = new ACE.Server.Entity.Spell(EnchantSpellId); if (!spell.NotFound) item.EnchantmentManager.Add(spell, item, null, false); } catch (Exception ex) { ModManager.Log($"[{Mod.Name}] mule-test could not enchant 0x{item.Guid.Full:X8}: {ex.Message}", ModManager.LogLevel.Warn); } if (!pack.TryAddToInventory(item, out _, pack.Inventory.Count, true, false)) { item.Destroy(); failed++; continue; } if (!SaveNow(item)) { failed++; continue; } Record(accountId, item); made++; } SaveNow(pack); return $"made {made} item(s) in the stand-in pack 0x{pack.Guid.Full:X8} ({pack.Inventory.Count} in it now){(failed > 0 ? $", {failed} failed" : "")}"; } // ---- moving -------------------------------------------------------------- /// /// Deposits up to items from the stand-in pack, through the /// same core the /mule command uses, one at a time, each with its own commit and its /// own ledger row. Returns a line of counts on both sides. /// public static string Deposit(uint accountId, int count, string actor = "mule-test") { var handle = WaitForVault(accountId, out var error); if (handle is null) return $"no vault: {error}"; var pack = Pack(accountId, out error); if (pack is null) return $"no stand-in pack: {error}"; var moved = 0; var refused = 0; var written = 0; var full = false; var entries = new List(); var committed = new List(); using (Vault.Hold(accountId)) { if (!Vault.IsCurrent(handle)) return "the vault was released between reading it and writing to it - nothing was changed"; foreach (var item in pack.Inventory.Values.ToList()) { if (moved >= count) break; if (handle.Count >= Mod.Settings.MaxItems) { entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId, item.Name ?? "", item.StackSize ?? 1, Movement.In, "refused:full", $"{handle.Count} of {Mod.Settings.MaxItems}", 0)); full = true; refused++; break; } var link = Vault.LinkWithRoom(handle, out var growError); if (link is null) { entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId, item.Name ?? "", item.StackSize ?? 1, Movement.In, growError == "full" ? "refused:full" : "refused:no-link", growError, 0)); full = growError == "full"; refused++; break; } if (!Movement.MoveIntoVault(pack, item, link, actor, out var why)) { entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId, item.Name ?? "", item.StackSize ?? 1, Movement.In, why, "", link.Guid.Full)); refused++; continue; } // THE COMMIT, per item and waited on, before the next item is touched. // Batching the writes to the end of the loop would also be safe - each // item's row still flips exactly once - but it would mean a kill during the // loop found nothing written at all, and the window this harness exists to // shoot at is the one where some items are committed and one is in flight. if (SaveNowAlreadyBookkept(item)) written++; entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId, item.Name ?? "", item.StackSize ?? 1, Movement.In, "moved", "", link.Guid.Full)); MuleDb.Write(entries[^1]); committed.Add(item); moved++; } } foreach (var refusal in entries.Where(e => e.Result != "moved").ToList()) MuleDb.Write(refusal); SaveNow(pack); return $"deposited {moved} ({written} write(s) confirmed), {refused} refused{(full ? " (vault full)" : "")}; " + $"vault {handle.Count}/{Mod.Settings.MaxItems}, stand-in pack {pack.Inventory.Count}"; } /// /// The commit for an item whose SaveBiotaToDatabase(false) bookkeeping was already done /// inside MoveIntoVault. Waited on, so a test knows the row landed. /// private static bool SaveNowAlreadyBookkept(WorldObject wo) { var ok = false; var done = new ManualResetEventSlim(false); DatabaseManager.Shard.SaveBiota(wo.Biota, wo.BiotaDatabaseLock, r => { ok = r; done.Set(); }); return done.Wait(TimeSpan.FromSeconds(30)) && ok; } public static string Withdraw(uint accountId, int count, string actor = "mule-test") { var handle = WaitForVault(accountId, out var error); if (handle is null) return $"no vault: {error}"; var pack = Pack(accountId, out error); if (pack is null) return $"no stand-in pack: {error}"; var moved = 0; var refused = 0; var written = 0; var entries = new List(); var committed = new List(); using (Vault.Hold(accountId)) { if (!Vault.IsCurrent(handle)) return "the vault was released between reading it and writing to it - nothing was changed"; foreach (var link in handle.Links) { foreach (var item in link.Inventory.Values.ToList()) { if (moved >= count) break; if (pack.Inventory.Count >= Mod.Settings.SlotsPerContainer) { refused++; break; } if (!link.TryRemoveFromInventory(item.Guid, out var taken)) { entries.Add(Line(accountId, actor, item, Movement.Out, "refused:not-removed", link.Guid.Full)); refused++; continue; } if (Movement.DebugPauseAfterRemoveMs > 0) Thread.Sleep(Movement.DebugPauseAfterRemoveMs); if (!pack.TryAddToInventory(taken, out _, pack.Inventory.Count, true, false)) { if (!link.TryAddToInventory(taken, out _, link.Inventory.Count, true, false)) { ModManager.Log($"[{Mod.Name}] STRANDED in mule-test: 0x{taken.Guid.Full:X8} - nothing was written.", ModManager.LogLevel.Error); entries.Add(Line(accountId, actor, taken, Movement.Out, "stranded", link.Guid.Full)); refused++; continue; } entries.Add(Line(accountId, actor, taken, Movement.Out, "refused:no-room", link.Guid.Full)); refused++; continue; } if (Movement.DebugPauseAfterAddMs > 0) Thread.Sleep(Movement.DebugPauseAfterAddMs); taken.SaveBiotaToDatabase(false); if (SaveNowAlreadyBookkept(taken)) written++; entries.Add(Line(accountId, actor, taken, Movement.Out, "moved", link.Guid.Full)); MuleDb.Write(entries[^1]); committed.Add(taken); moved++; } if (moved >= count) break; } } foreach (var refusal in entries.Where(e => e.Result != "moved").ToList()) MuleDb.Write(refusal); SaveNow(pack); return $"withdrew {moved} ({written} write(s) confirmed), {refused} refused; " + $"vault {handle.Count}/{Mod.Settings.MaxItems}, stand-in pack {pack.Inventory.Count}"; } private static MuleDb.Entry Line(uint accountId, string actor, WorldObject item, string direction, string result, uint containerId) => new(accountId, 0, actor, item.Guid.Full, item.WeenieClassId, item.Name ?? "", item.StackSize ?? 1, direction, result, "", containerId); // ---- the race ------------------------------------------------------------ /// /// Two threads deposit at the same time, on one account, WITHOUT marshalling onto the /// world thread - which is how Aeshnidae.RemoteConsole invokes a command handler, and /// so a real way for two movements to collide. /// /// The account lock is the only thing standing between them. Afterwards the invariant /// that must hold is exact: items deposited == items that left the stand-in pack == /// items the shard says are in the vault, and the lock must have been contended at /// least once, or the test proved nothing. /// public static string Race(uint accountId, int each) { var handle = WaitForVault(accountId, out var error); if (handle is null) return $"no vault: {error}"; var pack = Pack(accountId, out error); if (pack is null) return $"no stand-in pack: {error}"; var before = handle.Count; var beforePack = pack.Inventory.Count; var waitsBefore = Vault.LockWaits; var results = new string[2]; var gate = new ManualResetEventSlim(false); var a = new Thread(() => { gate.Wait(); results[0] = Deposit(accountId, each, "race-a"); }) { IsBackground = true, Name = "mule-race-a" }; var b = new Thread(() => { gate.Wait(); results[1] = Deposit(accountId, each, "race-b"); }) { IsBackground = true, Name = "mule-race-b" }; a.Start(); b.Start(); gate.Set(); a.Join(TimeSpan.FromMinutes(2)); b.Join(TimeSpan.FromMinutes(2)); var after = handle.Count; var afterPack = pack.Inventory.Count; var onDisk = MuleDb.CountInContainers(handle.Links.Select(l => l.Guid.Full)); var waits = Vault.LockWaits - waitsBefore; var movedIn = after - before; var leftPack = beforePack - afterPack; var consistent = movedIn == leftPack && after == onDisk; return $"race: a[{results[0]}] b[{results[1]}]\n" + $" vault {before} -> {after} (+{movedIn}), stand-in pack {beforePack} -> {afterPack} (-{leftPack}), shard says {onDisk}\n" + $" lock contended {waits} time(s)\n" + $" {(consistent ? "CONSISTENT" : "INCONSISTENT - items are not accounted for")}" + $"{(waits == 0 ? " (but the lock was never contended, so this run proves nothing about it)" : "")}"; } // ---- counting and accounting -------------------------------------------- public static string Count(uint accountId) { var handle = WaitForVault(accountId, out var error); if (handle is null) return $"no vault: {error}"; var pack = Pack(accountId, out _); var onDisk = MuleDb.CountInContainers(handle.Links.Select(l => l.Guid.Full)); var packDisk = pack is null ? 0 : MuleDb.CountInContainers(new[] { pack.Guid.Full }); return $"vault: memory {handle.Count}, shard {onDisk} across {handle.Links.Count} link(s) " + $"[{string.Join(", ", handle.Links.Select(l => $"0x{l.Guid.Full:X8}:{l.Inventory.Count}"))}]\n" + $"stand-in pack: memory {(pack?.Inventory.Count ?? -1)}, shard {packDisk}" + (pack is null ? "" : $" (0x{pack.Guid.Full:X8})") + "\n" + $"{(handle.Count == onDisk ? "MEMORY AND SHARD AGREE" : "MEMORY AND SHARD DISAGREE")}"; } /// /// The accounting that makes a crash test a fact. For every object id this harness ever /// created, the shard is asked where it is: a vault link, the stand-in pack, somewhere /// else, nowhere, or gone. None lost means nothing in `missing` or `nowhere`; none /// duplicated means the per-place counts add up to the total created. /// public static string Audit(uint accountId) { EnsureTables(); var vaultLinks = MuleDb.Chain(accountId).Select(r => r.ContainerId).ToHashSet(); uint packGuid = 0; using (var conn = Open()) using (var cmd = conn.CreateCommand()) { cmd.CommandText = $"SELECT `container_Id` FROM `{PackTable}` WHERE `account_Id` = @a;"; cmd.Parameters.AddWithValue("@a", accountId); var r = cmd.ExecuteScalar(); if (r is not (null or DBNull)) packGuid = Convert.ToUInt32(r); } var created = new List<(uint Id, uint Wcid, string Name)>(); using (var conn = Open()) using (var cmd = conn.CreateCommand()) { cmd.CommandText = $"SELECT `object_Id`, `wcid`, `name` FROM `{ItemTable}` WHERE `account_Id` = @a;"; cmd.Parameters.AddWithValue("@a", accountId); using var rd = cmd.ExecuteReader(); while (rd.Read()) created.Add((rd.GetUInt32(0), rd.GetUInt32(1), rd.GetString(2))); } int inVault = 0, inPack = 0, elsewhere = 0, nowhere = 0, missing = 0; var problems = new List(); foreach (var item in created) { var where = MuleDb.WhereIs(item.Id); if (!where.Exists) { missing++; problems.Add($"0x{item.Id:X8} {item.Name}: NO BIOTA ROW - the item is gone"); continue; } if (where.Container != 0 && vaultLinks.Contains(where.Container)) inVault++; else if (where.Container != 0 && where.Container == packGuid) inPack++; else if (where.Container != 0 || where.Wielder != 0 || where.HasLocation) { elsewhere++; problems.Add($"0x{item.Id:X8} {item.Name}: container 0x{where.Container:X8}, wielder 0x{where.Wielder:X8}, location {where.HasLocation}"); } else { nowhere++; problems.Add($"0x{item.Id:X8} {item.Name}: NO container, NO wielder, NO location - orphaned"); } } var placed = inVault + inPack + elsewhere + nowhere; var verdict = missing == 0 && nowhere == 0 && elsewhere == 0 && placed == created.Count ? "EVERY ITEM ACCOUNTED FOR: none lost, none orphaned, each in exactly one place" : "NOT ACCOUNTED FOR - see the lines above"; var text = new StringBuilder(); text.AppendLine($"account {accountId}: {created.Count} item(s) ever created by the harness"); text.AppendLine($" in a vault link : {inVault}"); text.AppendLine($" in the stand-in : {inPack}"); text.AppendLine($" somewhere else : {elsewhere}"); text.AppendLine($" nowhere : {nowhere}"); text.AppendLine($" no biota row : {missing}"); foreach (var p in problems.Take(25)) text.AppendLine(" ! " + p); if (problems.Count > 25) text.AppendLine($" ... and {problems.Count - 25} more"); text.Append(verdict); return text.ToString(); } /// /// Destroys the stand-in pack and everything in it, and forgets the records. Explicit, /// never automatic: a test that cleans up after itself cannot be audited after a crash. /// public static string Clear(uint accountId) { EnsureTables(); var pack = Pack(accountId, out var error); var destroyed = 0; if (pack is not null) { foreach (var item in pack.Inventory.Values.ToList()) { if (pack.TryRemoveFromInventory(item.Guid, out var taken)) { taken.Destroy(); destroyed++; } } pack.Destroy(); _packs.TryRemove(accountId, out _); } using (var conn = Open()) { using var a = conn.CreateCommand(); a.CommandText = $"DELETE FROM `{PackTable}` WHERE `account_Id` = @a;"; a.Parameters.AddWithValue("@a", accountId); a.ExecuteNonQuery(); using var b = conn.CreateCommand(); b.CommandText = $"DELETE FROM `{ItemTable}` WHERE `account_Id` = @a;"; b.Parameters.AddWithValue("@a", accountId); b.ExecuteNonQuery(); } return $"stand-in pack destroyed, {destroyed} item(s) with it{(error.Length > 0 ? $" ({error})" : "")}"; } /// Waits for a vault to finish loading - the harness is never on the world thread. public static VaultHandle? WaitForVault(uint accountId, out string error) { Vault.Warm(accountId); var deadline = DateTime.UtcNow.AddSeconds(Mod.Settings.LoadTimeoutSeconds + 10); while (DateTime.UtcNow < deadline) { var handle = Vault.Get(accountId, out error); if (handle is not null) return handle; if (error.Contains("could not", StringComparison.OrdinalIgnoreCase) || error.Contains("too long", StringComparison.OrdinalIgnoreCase)) return null; Thread.Sleep(50); } error = "the vault did not finish loading"; return null; } }