namespace Aeshnidae.Mule;
///
/// The console test harness. STAGING SCAFFOLDING: every verb refuses unless
/// Settings.AllowTestHarness is true, which it is not by default and must never be on
/// live - these commands create and destroy real items.
///
/// Why it exists, and what it does and does not prove
/// --------------------------------------------------
/// A vault has to be tested with hundreds of items, with the server killed mid-flight, and
/// with two movements racing on one account. None of that can be done by hand in a client,
/// and the parts that matter are not client-side: they are the ordering of the three steps,
/// the single commit, the account lock and the accounting afterwards.
///
/// So the harness drives the SAME core - Movement.MoveIntoVault, the same account lock, the
/// same single confirmed write, the same ledger - with a STAND-IN PACK in place of a
/// player's inventory. A stand-in pack is a real Container holding real WorldObjects with
/// real biotas, so every database fact the real path produces, this produces. What it does
/// NOT exercise is the Player half: the refusals that need a live session (trade window,
/// wielded, too busy) and the client messages. Those are listed as unverified.
///
/// It also keeps its own durable record of every item it created
/// (`aeshnidae_mule_test_item`), which is what makes the crash test an accounting exercise
/// rather than an opinion: after a SIGKILL and a restart, every object id it ever minted
/// must be in exactly one place.
///
public static class MuleTest
{
public const string PackTable = "aeshnidae_mule_test";
public const string ItemTable = "aeshnidae_mule_test_item";
/// Impenetrability VI - an ordinary item enchantment, for the fidelity test.
private const uint EnchantSpellId = 1486;
private static string _cs = "";
public static void Initialize()
{
var cfg = ConfigManager.Config?.MySql?.Shard;
if (cfg is null)
return;
_cs = new MySqlConnectionStringBuilder
{
Server = cfg.Host,
Port = cfg.Port,
Database = cfg.Database,
UserID = cfg.Username,
Password = cfg.Password,
}.ConnectionString;
}
private static MySqlConnection Open()
{
var conn = new MySqlConnection(_cs);
conn.Open();
return conn;
}
private static void EnsureTables()
{
using var conn = Open();
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = $@"
CREATE TABLE IF NOT EXISTS `{PackTable}` (
`account_Id` INT UNSIGNED NOT NULL,
`container_Id` INT UNSIGNED NOT NULL,
PRIMARY KEY (`account_Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;";
cmd.ExecuteNonQuery();
}
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = $@"
CREATE TABLE IF NOT EXISTS `{ItemTable}` (
`account_Id` INT UNSIGNED NOT NULL,
`object_Id` INT UNSIGNED NOT NULL,
`wcid` INT UNSIGNED NOT NULL DEFAULT 0,
`name` VARCHAR(128) NOT NULL DEFAULT '',
PRIMARY KEY (`object_Id`),
KEY `k_account` (`account_Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;";
cmd.ExecuteNonQuery();
}
}
// ---- the stand-in pack ---------------------------------------------------
private static readonly ConcurrentDictionary _packs = new();
///
/// The account's stand-in pack, loaded or minted. Same shape as a vault link and saved
/// the same way, so it survives a kill and can be counted afterwards.
///
public static Container? Pack(uint accountId, out string error)
{
error = "";
EnsureTables();
if (_packs.TryGetValue(accountId, out var cached))
return cached;
uint guid = 0;
using (var conn = Open())
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = $"SELECT `container_Id` FROM `{PackTable}` WHERE `account_Id` = @a;";
cmd.Parameters.AddWithValue("@a", accountId);
var r = cmd.ExecuteScalar();
if (r is not (null or DBNull))
guid = Convert.ToUInt32(r);
}
ShardBiota? dbBiota = guid == 0 ? null : DatabaseManager.Shard.BaseDatabase.GetBiota(guid);
if (guid != 0)
{
if (dbBiota is null)
{
// Harness scaffolding, not a vault: if something removed it - the
// orphaned-biota purge test does exactly that, since a stand-in pack is
// deliberately NOT protected - forget the row and mint a new one. A real
// vault link in this state is refused instead and needs a human, because
// items may be pointing at it (Vault.LoadNow).
ModManager.Log($"[{Mod.Name}] mule-test: the stand-in pack 0x{guid:X8} has no biota row; forgetting it and minting a new one.",
ModManager.LogLevel.Warn);
using (var conn = Open())
using (var del = conn.CreateCommand())
{
del.CommandText = $"DELETE FROM `{PackTable}` WHERE `account_Id` = @a;";
del.Parameters.AddWithValue("@a", accountId);
del.ExecuteNonQuery();
}
guid = 0;
}
}
if (guid != 0 && dbBiota is not null)
{
if (WorldObjectFactory.CreateWorldObject(dbBiota) is not Container loaded)
{
error = $"0x{guid:X8} is not a container";
return null;
}
var deadline = DateTime.UtcNow.AddSeconds(Mod.Settings.LoadTimeoutSeconds);
while (!loaded.InventoryLoaded)
{
if (DateTime.UtcNow > deadline)
{
error = "the stand-in pack did not finish loading";
return null;
}
Thread.Sleep(25);
}
_packs[accountId] = loaded;
return loaded;
}
var weenie = DatabaseManager.World.GetCachedWeenie(Mod.Settings.VaultWcid);
if (weenie is null)
{
error = $"weenie {Mod.Settings.VaultWcid} is not in the world database";
return null;
}
var fresh = WorldObjectFactory.CreateWorldObject(weenie, GuidManager.NewDynamicGuid()) as Container;
if (fresh is null)
{
error = "the stand-in pack could not be created";
return null;
}
fresh.ItemCapacity = (byte)Mod.Settings.SlotsPerContainer;
fresh.ContainerCapacity = 0;
if (!SaveNow(fresh))
{
error = "the stand-in pack could not be saved";
return null;
}
using (var conn = Open())
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = $"INSERT INTO `{PackTable}` (`account_Id`, `container_Id`) VALUES (@a, @c) ON DUPLICATE KEY UPDATE `container_Id` = @c;";
cmd.Parameters.AddWithValue("@a", accountId);
cmd.Parameters.AddWithValue("@c", fresh.Guid.Full);
cmd.ExecuteNonQuery();
}
_packs[accountId] = fresh;
return fresh;
}
/// A save that is waited on, so a test never races the database.
public static bool SaveNow(WorldObject wo)
{
var ok = false;
var done = new ManualResetEventSlim(false);
wo.SaveBiotaToDatabase(false);
DatabaseManager.Shard.SaveBiota(wo.Biota, wo.BiotaDatabaseLock, r => { ok = r; done.Set(); });
return done.Wait(TimeSpan.FromSeconds(30)) && ok;
}
private static void Record(uint accountId, WorldObject item)
{
using var conn = Open();
using var cmd = conn.CreateCommand();
cmd.CommandText = $"INSERT IGNORE INTO `{ItemTable}` (`account_Id`, `object_Id`, `wcid`, `name`) VALUES (@a, @o, @w, @n);";
cmd.Parameters.AddWithValue("@a", accountId);
cmd.Parameters.AddWithValue("@o", item.Guid.Full);
cmd.Parameters.AddWithValue("@w", item.WeenieClassId);
cmd.Parameters.AddWithValue("@n", (item.Name ?? "").Length > 128 ? item.Name![..128] : item.Name ?? "");
cmd.ExecuteNonQuery();
}
// ---- filling -------------------------------------------------------------
///
/// Mints real items into the stand-in pack and records every
/// object id. Each is saved and confirmed before it counts as created, so the durable
/// record and the database never disagree about what exists.
///
public static string Fill(uint accountId, int count, uint wcid, int treasureProfile)
{
var pack = Pack(accountId, out var error);
if (pack is null)
return $"no stand-in pack: {error}";
var made = 0;
var failed = 0;
for (var i = 0; i < count; i++)
{
if (pack.Inventory.Count >= Mod.Settings.SlotsPerContainer)
return $"the stand-in pack is full at {pack.Inventory.Count} - deposit what is in it first. {made} made.";
WorldObject? item = null;
if (treasureProfile > 0)
{
var profile = DatabaseManager.World.GetCachedDeathTreasure((uint)treasureProfile);
if (profile is null)
return $"treasure profile {treasureProfile} is not in the world database";
item = LootGenerationFactory.CreateRandomLootObjects_Test(profile, true);
}
else
{
item = WorldObjectFactory.CreateNewWorldObject(wcid);
}
if (item is null) { failed++; continue; }
// Stamp things a round trip must not lose, so the before/after comparison has
// something to compare: our own 9042/9043, workmanship, tinks and item XP.
item.SetProperty((PropertyInt)9043, 7);
item.SetProperty((PropertyInt)9042, 1);
if (item.ItemWorkmanship is null)
item.SetProperty(PropertyInt.ItemWorkmanship, 7);
item.SetProperty(PropertyInt.NumTimesTinkered, 3);
item.SetProperty(PropertyInt.ImbuedEffect, (int)ImbuedEffectType.CriticalStrike);
item.SetProperty(PropertyInt64.ItemTotalXp, 1_234_567L);
item.SetProperty(PropertyString.Inscription, $"mule test {accountId} #{i}");
// A live enchantment ON THE ITEM, in its own enchantment registry - the one
// thing a vault cannot keep ticking (no landblock means no heartbeat), so the
// round trip has to prove at least that the row itself survives untouched.
try
{
var spell = new ACE.Server.Entity.Spell(EnchantSpellId);
if (!spell.NotFound)
item.EnchantmentManager.Add(spell, item, null, false);
}
catch (Exception ex)
{
ModManager.Log($"[{Mod.Name}] mule-test could not enchant 0x{item.Guid.Full:X8}: {ex.Message}", ModManager.LogLevel.Warn);
}
if (!pack.TryAddToInventory(item, out _, pack.Inventory.Count, true, false))
{
item.Destroy();
failed++;
continue;
}
if (!SaveNow(item))
{
failed++;
continue;
}
Record(accountId, item);
made++;
}
SaveNow(pack);
return $"made {made} item(s) in the stand-in pack 0x{pack.Guid.Full:X8} ({pack.Inventory.Count} in it now){(failed > 0 ? $", {failed} failed" : "")}";
}
// ---- moving --------------------------------------------------------------
///
/// Deposits up to items from the stand-in pack, through the
/// same core the /mule command uses, one at a time, each with its own commit and its
/// own ledger row. Returns a line of counts on both sides.
///
public static string Deposit(uint accountId, int count, string actor = "mule-test")
{
var handle = WaitForVault(accountId, out var error);
if (handle is null)
return $"no vault: {error}";
var pack = Pack(accountId, out error);
if (pack is null)
return $"no stand-in pack: {error}";
var moved = 0;
var refused = 0;
var written = 0;
var full = false;
var entries = new List();
var committed = new List();
using (Vault.Hold(accountId))
{
if (!Vault.IsCurrent(handle))
return "the vault was released between reading it and writing to it - nothing was changed";
foreach (var item in pack.Inventory.Values.ToList())
{
if (moved >= count)
break;
if (handle.Count >= Mod.Settings.MaxItems)
{
entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId,
item.Name ?? "", item.StackSize ?? 1, Movement.In, "refused:full",
$"{handle.Count} of {Mod.Settings.MaxItems}", 0));
full = true;
refused++;
break;
}
var link = Vault.LinkWithRoom(handle, out var growError);
if (link is null)
{
entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId,
item.Name ?? "", item.StackSize ?? 1, Movement.In,
growError == "full" ? "refused:full" : "refused:no-link", growError, 0));
full = growError == "full";
refused++;
break;
}
if (!Movement.MoveIntoVault(pack, item, link, actor, out var why))
{
entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId,
item.Name ?? "", item.StackSize ?? 1, Movement.In, why, "", link.Guid.Full));
refused++;
continue;
}
// THE COMMIT, per item and waited on, before the next item is touched.
// Batching the writes to the end of the loop would also be safe - each
// item's row still flips exactly once - but it would mean a kill during the
// loop found nothing written at all, and the window this harness exists to
// shoot at is the one where some items are committed and one is in flight.
if (SaveNowAlreadyBookkept(item))
written++;
entries.Add(new MuleDb.Entry(accountId, 0, actor, item.Guid.Full, item.WeenieClassId,
item.Name ?? "", item.StackSize ?? 1, Movement.In, "moved", "", link.Guid.Full));
MuleDb.Write(entries[^1]);
committed.Add(item);
moved++;
}
}
foreach (var refusal in entries.Where(e => e.Result != "moved").ToList())
MuleDb.Write(refusal);
SaveNow(pack);
return $"deposited {moved} ({written} write(s) confirmed), {refused} refused{(full ? " (vault full)" : "")}; " +
$"vault {handle.Count}/{Mod.Settings.MaxItems}, stand-in pack {pack.Inventory.Count}";
}
///
/// The commit for an item whose SaveBiotaToDatabase(false) bookkeeping was already done
/// inside MoveIntoVault. Waited on, so a test knows the row landed.
///
private static bool SaveNowAlreadyBookkept(WorldObject wo)
{
var ok = false;
var done = new ManualResetEventSlim(false);
DatabaseManager.Shard.SaveBiota(wo.Biota, wo.BiotaDatabaseLock, r => { ok = r; done.Set(); });
return done.Wait(TimeSpan.FromSeconds(30)) && ok;
}
public static string Withdraw(uint accountId, int count, string actor = "mule-test")
{
var handle = WaitForVault(accountId, out var error);
if (handle is null)
return $"no vault: {error}";
var pack = Pack(accountId, out error);
if (pack is null)
return $"no stand-in pack: {error}";
var moved = 0;
var refused = 0;
var written = 0;
var entries = new List();
var committed = new List();
using (Vault.Hold(accountId))
{
if (!Vault.IsCurrent(handle))
return "the vault was released between reading it and writing to it - nothing was changed";
foreach (var link in handle.Links)
{
foreach (var item in link.Inventory.Values.ToList())
{
if (moved >= count)
break;
if (pack.Inventory.Count >= Mod.Settings.SlotsPerContainer)
{
refused++;
break;
}
if (!link.TryRemoveFromInventory(item.Guid, out var taken))
{
entries.Add(Line(accountId, actor, item, Movement.Out, "refused:not-removed", link.Guid.Full));
refused++;
continue;
}
if (Movement.DebugPauseAfterRemoveMs > 0)
Thread.Sleep(Movement.DebugPauseAfterRemoveMs);
if (!pack.TryAddToInventory(taken, out _, pack.Inventory.Count, true, false))
{
if (!link.TryAddToInventory(taken, out _, link.Inventory.Count, true, false))
{
ModManager.Log($"[{Mod.Name}] STRANDED in mule-test: 0x{taken.Guid.Full:X8} - nothing was written.", ModManager.LogLevel.Error);
entries.Add(Line(accountId, actor, taken, Movement.Out, "stranded", link.Guid.Full));
refused++;
continue;
}
entries.Add(Line(accountId, actor, taken, Movement.Out, "refused:no-room", link.Guid.Full));
refused++;
continue;
}
if (Movement.DebugPauseAfterAddMs > 0)
Thread.Sleep(Movement.DebugPauseAfterAddMs);
taken.SaveBiotaToDatabase(false);
if (SaveNowAlreadyBookkept(taken))
written++;
entries.Add(Line(accountId, actor, taken, Movement.Out, "moved", link.Guid.Full));
MuleDb.Write(entries[^1]);
committed.Add(taken);
moved++;
}
if (moved >= count)
break;
}
}
foreach (var refusal in entries.Where(e => e.Result != "moved").ToList())
MuleDb.Write(refusal);
SaveNow(pack);
return $"withdrew {moved} ({written} write(s) confirmed), {refused} refused; " +
$"vault {handle.Count}/{Mod.Settings.MaxItems}, stand-in pack {pack.Inventory.Count}";
}
private static MuleDb.Entry Line(uint accountId, string actor, WorldObject item, string direction, string result, uint containerId) =>
new(accountId, 0, actor, item.Guid.Full, item.WeenieClassId, item.Name ?? "", item.StackSize ?? 1,
direction, result, "", containerId);
// ---- the race ------------------------------------------------------------
///
/// Two threads deposit at the same time, on one account, WITHOUT marshalling onto the
/// world thread - which is how Aeshnidae.RemoteConsole invokes a command handler, and
/// so a real way for two movements to collide.
///
/// The account lock is the only thing standing between them. Afterwards the invariant
/// that must hold is exact: items deposited == items that left the stand-in pack ==
/// items the shard says are in the vault, and the lock must have been contended at
/// least once, or the test proved nothing.
///
public static string Race(uint accountId, int each)
{
var handle = WaitForVault(accountId, out var error);
if (handle is null)
return $"no vault: {error}";
var pack = Pack(accountId, out error);
if (pack is null)
return $"no stand-in pack: {error}";
var before = handle.Count;
var beforePack = pack.Inventory.Count;
var waitsBefore = Vault.LockWaits;
var results = new string[2];
var gate = new ManualResetEventSlim(false);
var a = new Thread(() => { gate.Wait(); results[0] = Deposit(accountId, each, "race-a"); }) { IsBackground = true, Name = "mule-race-a" };
var b = new Thread(() => { gate.Wait(); results[1] = Deposit(accountId, each, "race-b"); }) { IsBackground = true, Name = "mule-race-b" };
a.Start();
b.Start();
gate.Set();
a.Join(TimeSpan.FromMinutes(2));
b.Join(TimeSpan.FromMinutes(2));
var after = handle.Count;
var afterPack = pack.Inventory.Count;
var onDisk = MuleDb.CountInContainers(handle.Links.Select(l => l.Guid.Full));
var waits = Vault.LockWaits - waitsBefore;
var movedIn = after - before;
var leftPack = beforePack - afterPack;
var consistent = movedIn == leftPack && after == onDisk;
return $"race: a[{results[0]}] b[{results[1]}]\n" +
$" vault {before} -> {after} (+{movedIn}), stand-in pack {beforePack} -> {afterPack} (-{leftPack}), shard says {onDisk}\n" +
$" lock contended {waits} time(s)\n" +
$" {(consistent ? "CONSISTENT" : "INCONSISTENT - items are not accounted for")}" +
$"{(waits == 0 ? " (but the lock was never contended, so this run proves nothing about it)" : "")}";
}
// ---- counting and accounting --------------------------------------------
public static string Count(uint accountId)
{
var handle = WaitForVault(accountId, out var error);
if (handle is null)
return $"no vault: {error}";
var pack = Pack(accountId, out _);
var onDisk = MuleDb.CountInContainers(handle.Links.Select(l => l.Guid.Full));
var packDisk = pack is null ? 0 : MuleDb.CountInContainers(new[] { pack.Guid.Full });
return $"vault: memory {handle.Count}, shard {onDisk} across {handle.Links.Count} link(s) " +
$"[{string.Join(", ", handle.Links.Select(l => $"0x{l.Guid.Full:X8}:{l.Inventory.Count}"))}]\n" +
$"stand-in pack: memory {(pack?.Inventory.Count ?? -1)}, shard {packDisk}" +
(pack is null ? "" : $" (0x{pack.Guid.Full:X8})") + "\n" +
$"{(handle.Count == onDisk ? "MEMORY AND SHARD AGREE" : "MEMORY AND SHARD DISAGREE")}";
}
///
/// The accounting that makes a crash test a fact. For every object id this harness ever
/// created, the shard is asked where it is: a vault link, the stand-in pack, somewhere
/// else, nowhere, or gone. None lost means nothing in `missing` or `nowhere`; none
/// duplicated means the per-place counts add up to the total created.
///
public static string Audit(uint accountId)
{
EnsureTables();
var vaultLinks = MuleDb.Chain(accountId).Select(r => r.ContainerId).ToHashSet();
uint packGuid = 0;
using (var conn = Open())
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = $"SELECT `container_Id` FROM `{PackTable}` WHERE `account_Id` = @a;";
cmd.Parameters.AddWithValue("@a", accountId);
var r = cmd.ExecuteScalar();
if (r is not (null or DBNull))
packGuid = Convert.ToUInt32(r);
}
var created = new List<(uint Id, uint Wcid, string Name)>();
using (var conn = Open())
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = $"SELECT `object_Id`, `wcid`, `name` FROM `{ItemTable}` WHERE `account_Id` = @a;";
cmd.Parameters.AddWithValue("@a", accountId);
using var rd = cmd.ExecuteReader();
while (rd.Read())
created.Add((rd.GetUInt32(0), rd.GetUInt32(1), rd.GetString(2)));
}
int inVault = 0, inPack = 0, elsewhere = 0, nowhere = 0, missing = 0;
var problems = new List();
foreach (var item in created)
{
var where = MuleDb.WhereIs(item.Id);
if (!where.Exists)
{
missing++;
problems.Add($"0x{item.Id:X8} {item.Name}: NO BIOTA ROW - the item is gone");
continue;
}
if (where.Container != 0 && vaultLinks.Contains(where.Container)) inVault++;
else if (where.Container != 0 && where.Container == packGuid) inPack++;
else if (where.Container != 0 || where.Wielder != 0 || where.HasLocation)
{
elsewhere++;
problems.Add($"0x{item.Id:X8} {item.Name}: container 0x{where.Container:X8}, wielder 0x{where.Wielder:X8}, location {where.HasLocation}");
}
else
{
nowhere++;
problems.Add($"0x{item.Id:X8} {item.Name}: NO container, NO wielder, NO location - orphaned");
}
}
var placed = inVault + inPack + elsewhere + nowhere;
var verdict = missing == 0 && nowhere == 0 && elsewhere == 0 && placed == created.Count
? "EVERY ITEM ACCOUNTED FOR: none lost, none orphaned, each in exactly one place"
: "NOT ACCOUNTED FOR - see the lines above";
var text = new StringBuilder();
text.AppendLine($"account {accountId}: {created.Count} item(s) ever created by the harness");
text.AppendLine($" in a vault link : {inVault}");
text.AppendLine($" in the stand-in : {inPack}");
text.AppendLine($" somewhere else : {elsewhere}");
text.AppendLine($" nowhere : {nowhere}");
text.AppendLine($" no biota row : {missing}");
foreach (var p in problems.Take(25))
text.AppendLine(" ! " + p);
if (problems.Count > 25)
text.AppendLine($" ... and {problems.Count - 25} more");
text.Append(verdict);
return text.ToString();
}
///
/// Destroys the stand-in pack and everything in it, and forgets the records. Explicit,
/// never automatic: a test that cleans up after itself cannot be audited after a crash.
///
public static string Clear(uint accountId)
{
EnsureTables();
var pack = Pack(accountId, out var error);
var destroyed = 0;
if (pack is not null)
{
foreach (var item in pack.Inventory.Values.ToList())
{
if (pack.TryRemoveFromInventory(item.Guid, out var taken))
{
taken.Destroy();
destroyed++;
}
}
pack.Destroy();
_packs.TryRemove(accountId, out _);
}
using (var conn = Open())
{
using var a = conn.CreateCommand();
a.CommandText = $"DELETE FROM `{PackTable}` WHERE `account_Id` = @a;";
a.Parameters.AddWithValue("@a", accountId);
a.ExecuteNonQuery();
using var b = conn.CreateCommand();
b.CommandText = $"DELETE FROM `{ItemTable}` WHERE `account_Id` = @a;";
b.Parameters.AddWithValue("@a", accountId);
b.ExecuteNonQuery();
}
return $"stand-in pack destroyed, {destroyed} item(s) with it{(error.Length > 0 ? $" ({error})" : "")}";
}
/// Waits for a vault to finish loading - the harness is never on the world thread.
public static VaultHandle? WaitForVault(uint accountId, out string error)
{
Vault.Warm(accountId);
var deadline = DateTime.UtcNow.AddSeconds(Mod.Settings.LoadTimeoutSeconds + 10);
while (DateTime.UtcNow < deadline)
{
var handle = Vault.Get(accountId, out error);
if (handle is not null)
return handle;
if (error.Contains("could not", StringComparison.OrdinalIgnoreCase) ||
error.Contains("too long", StringComparison.OrdinalIgnoreCase))
return null;
Thread.Sleep(50);
}
error = "the vault did not finish loading";
return null;
}
}