namespace Aeshnidae.Mule; /// /// One account's vault, in memory: the chain of container WorldObjects and the items in /// them. Every item in here is the SAME WorldObject the player was carrying - the same /// guid, the same biota, the same spells, tinks, enchantments and item XP. A deposit is a /// change of container and nothing else. Nothing in this mod reads an item's properties /// into a row and rebuilds the item later, and nothing ever may. /// public sealed class VaultHandle { public uint AccountId { get; init; } /// Chain links in seq order. Index == seq. public List Links { get; } = new(); /// True once every link's inventory is back from the shard database. public volatile bool Ready; /// Set when a load failed; every verb refuses and says this. public volatile string Fault = ""; public DateTime LastTouched = DateTime.UtcNow; /// /// Items held, counted off the live containers. This is the authority while a vault is /// loaded; MuleDb.CountInContainers is the disk's opinion and the two are compared by /// `mule-admin verify`. /// public int Count => Links.Sum(l => l.Inventory.Count); public IEnumerable Items => Links.SelectMany(l => l.Inventory.Values); } /// /// Which vaults are loaded, how they are loaded, how they are kept on disk, and the lock /// that makes two clients on one account safe. /// /// WHY THIS IS NOT JUST A CONTAINER IN A LANDBLOCK /// ----------------------------------------------- /// A vault container is in no landblock, and that is deliberate - it must never be /// reachable, visible or openable. ACE's whole upkeep of an object runs off its landblock: /// Landblock.Tick walks a sorted heartbeat list that an object only joins through /// AddWorldObject, and Landblock.SaveDB only ever walks its own worldObjects. So a vault /// container gets NO heartbeat and NO automatic save, and this mod has to do both jobs: /// /// - saving: every movement forces the moved item's biota to disk before the player is /// told it worked (Movement), a timer sweeps for retries, and Dispose saves /// synchronously at shutdown. /// - the heartbeat we deliberately do NOT replace: enchantment durations on a stored /// item do not tick down, and an item with a Lifespan cannot expire in storage. /// Both are recorded in the Readme as what the vault does. /// /// Loading is ACE's own: Container's biota constructor asks the shard for everything whose /// container_Id is the container, and queues the sort onto WorldManager's action queue - /// which the world thread drains every tick whether or not any landblock is awake /// (WorldManager.UpdateGameWorld -> ActionQueue.RunActions). WorldObject.EnqueueAction has /// an explicit allowance for exactly this case. So the load works; it is just not instant, /// and a vault that has not finished loading is never treated as empty. /// public static class Vault { private static readonly ConcurrentDictionary _loaded = new(); private static readonly ConcurrentDictionary _gates = new(); private static readonly ConcurrentDictionary _loading = new(); /// How often a mutation had to wait for another one on the same account. public static int LockWaits; /// How many loads have been started, finished and failed, for mule-admin. public static int LoadsStarted, LoadsFinished, LoadsFailed; /// Items saved by the safety-net sweep, and rows it wrote. public static int SweepSaves; /// /// THE ACCOUNT LOCK. Every mutation of a vault, and every read that must agree with a /// mutation (the count behind the limit, the sweep's walk of Inventory), is done inside /// `lock (Vault.Gate(accountId))`. /// /// Two clients on one account is normal here - FairPlay's per-address caps are off for /// the beta - so two simultaneous deposits are the obvious race. Player commands /// happen to arrive on the world thread (InboundMessageManager's queue is drained in /// WorldManager.UpdateWorld), which would serialise them by itself; the console does /// NOT - Aeshnidae.RemoteConsole invokes a command handler straight off its own watcher /// thread - and nor does the save sweep. So the lock is load-bearing, not decoration, /// and `mule-admin race` exists to prove it holds. /// public static object Gate(uint accountId) => _gates.GetOrAdd(accountId, _ => new object()); /// Takes the account lock, counting the wait if there was one. public static IDisposable Hold(uint accountId) { var gate = Gate(accountId); if (!Monitor.TryEnter(gate)) { Interlocked.Increment(ref LockWaits); Monitor.Enter(gate); } return new Held(gate); } private sealed class Held : IDisposable { private readonly object _gate; private bool _done; public Held(object gate) => _gate = gate; public void Dispose() { if (_done) return; _done = true; Monitor.Exit(_gate); } } public static IEnumerable Loaded => _loaded.Values; public static int LoadedCount => _loaded.Count; /// /// The loaded vault for an account, or null with a reason. NEVER BLOCKS: this is called /// from the world thread, and the load it would be waiting for finishes ON the world /// thread, so waiting here would deadlock the server. A first call starts the load in /// the background and refuses; by the time a player can type a second command it is /// there. Login warms it (Patches), so in practice a player never sees the refusal. /// public static VaultHandle? Get(uint accountId, out string error) { error = ""; if (!Mod.Settings.Enabled) { error = "The vault is closed."; return null; } if (!MuleDb.Ready) { error = "The vault's ledger is not available right now. Nothing has been changed."; return null; } if (_loaded.TryGetValue(accountId, out var handle)) { handle.LastTouched = DateTime.UtcNow; if (handle.Fault.Length > 0) { error = handle.Fault; return null; } if (!handle.Ready) { error = "Your vault is still being read. Try again in a moment."; return null; } return handle; } Warm(accountId); error = "Your vault is being read. Try again in a moment."; return null; } /// /// Is this still THE loaded vault for its account? /// /// Get() cannot take the account lock - it is called from the world thread and the load /// it would wait for finishes on that thread - so there is a sliver between a caller /// getting a handle and taking the lock in which the sweep could have released that /// vault. Nothing on disk could go wrong (the item's own row is still the truth, and the /// sweep saves before letting go), but the released handle's in-memory count would /// diverge from a freshly loaded one, and the limit is counted from that count. So every /// mutation re-checks this as its first act inside the lock, and refuses if it lost the /// race rather than working from a count that may be stale. /// public static bool IsCurrent(VaultHandle handle) => handle is not null && _loaded.TryGetValue(handle.AccountId, out var current) && ReferenceEquals(current, handle); /// /// Starts loading an account's vault if it is not loaded or loading. Safe to call from /// any thread and as often as you like. Called on login, so the first /mule is ready. /// public static void Warm(uint accountId) { if (!Mod.Settings.Enabled || !MuleDb.Ready || accountId == 0) return; if (_loaded.ContainsKey(accountId)) { if (_loaded.TryGetValue(accountId, out var h)) h.LastTouched = DateTime.UtcNow; return; } if (!_loading.TryAdd(accountId, 0)) return; Interlocked.Increment(ref LoadsStarted); Task.Run(() => { try { LoadNow(accountId); } catch (Exception ex) { Interlocked.Increment(ref LoadsFailed); ModManager.Log($"[{Mod.Name}] loading the vault of account {accountId} threw: {ex}", ModManager.LogLevel.Error); _loaded[accountId] = new VaultHandle { AccountId = accountId, Fault = "Your vault could not be read. It has not been changed - tell a developer.", }; } finally { _loading.TryRemove(accountId, out _); } }); } /// /// Reads an account's chain off the shard and builds the live containers. Runs on a /// background task - never the world thread - because GetBiota blocks and because the /// sort of each container's contents has to run on the world thread while this waits. /// private static void LoadNow(uint accountId) { var rows = MuleDb.Chain(accountId); var handle = new VaultHandle { AccountId = accountId }; foreach (var row in rows) { var dbBiota = DatabaseManager.Shard.BaseDatabase.GetBiota(row.ContainerId); if (dbBiota is null) { // The row names a container that is not in the biota table. Do NOT invent // one: anything that was in it is pointing at a guid nothing owns, and // re-minting the container with that guid is a repair an operator should // choose (`mule-admin repair`), not something a login does silently. handle.Fault = "Your vault could not be opened. Nothing has been changed - tell a developer."; Interlocked.Increment(ref LoadsFailed); ModManager.Log($"[{Mod.Name}] VAULT LINK MISSING: account {accountId} link {row.Seq} names container 0x{row.ContainerId:X8}, which has no biota row. The vault is refused until `mule-admin repair {accountId}` is run deliberately.", ModManager.LogLevel.Error); _loaded[accountId] = handle; return; } if (dbBiota.WeenieClassId != Mod.Settings.VaultWcid) { // Worse: the guid belongs to something else now. Never touch it. handle.Fault = "Your vault could not be opened. Nothing has been changed - tell a developer."; Interlocked.Increment(ref LoadsFailed); ModManager.Log($"[{Mod.Name}] VAULT GUID COLLISION: account {accountId} link {row.Seq} names 0x{row.ContainerId:X8}, but that biota is weenie {dbBiota.WeenieClassId}, not the vault weenie {Mod.Settings.VaultWcid}. Refusing the whole vault.", ModManager.LogLevel.Error); _loaded[accountId] = handle; return; } if (WorldObjectFactory.CreateWorldObject(dbBiota) is not Container link) { handle.Fault = "Your vault could not be opened. Nothing has been changed - tell a developer."; Interlocked.Increment(ref LoadsFailed); ModManager.Log($"[{Mod.Name}] vault link 0x{row.ContainerId:X8} of account {accountId} did not come back as a Container.", ModManager.LogLevel.Error); _loaded[accountId] = handle; return; } handle.Links.Add(link); } // Publish it unready so a command refuses rather than reading a half-loaded vault. _loaded[accountId] = handle; var deadline = DateTime.UtcNow.AddSeconds(Mod.Settings.LoadTimeoutSeconds); while (handle.Links.Any(l => !l.InventoryLoaded)) { if (DateTime.UtcNow > deadline) { handle.Fault = "Your vault is taking too long to read. Nothing has been changed - try again shortly."; Interlocked.Increment(ref LoadsFailed); ModManager.Log($"[{Mod.Name}] the vault of account {accountId} did not finish loading inside {Mod.Settings.LoadTimeoutSeconds}s ({handle.Links.Count(l => !l.InventoryLoaded)} of {handle.Links.Count} link(s) still reading).", ModManager.LogLevel.Error); return; } Thread.Sleep(25); } handle.Ready = true; Interlocked.Increment(ref LoadsFinished); Mod.Trace($"vault of account {accountId} loaded: {handle.Links.Count} link(s), {handle.Count} item(s)"); } /// /// The link a deposit should go into: the first with a free slot. Null when the chain /// is as long as MaxItems allows and every link is full - the caller turns that into /// the limit refusal. Grows the chain when it needs to. Call under the account lock. /// public static Container? LinkWithRoom(VaultHandle handle, out string error) { error = ""; foreach (var link in handle.Links) { if (link.Inventory.Count < Mod.Settings.SlotsPerContainer) return link; } if (handle.Links.Count >= Mod.Settings.ChainLength) { error = "full"; return null; } return Grow(handle, out error); } /// /// Mints one more link: a container WorldObject of our hidden weenie, saved to the /// shard and CONFIRMED there before the row that claims it is written. /// /// The order is the safety. See MuleDb.AddLink for why this way round and not the /// other. Called under the account lock, on the world thread, so it blocks the world /// for one database round trip - once per 250 items an account ever deposits. /// private static Container? Grow(VaultHandle handle, out string error) { error = ""; var weenie = DatabaseManager.World.GetCachedWeenie(Mod.Settings.VaultWcid); if (weenie is null) { error = $"the vault weenie {Mod.Settings.VaultWcid} is not in the world database"; ModManager.Log($"[{Mod.Name}] {error} - no vault can be created", ModManager.LogLevel.Error); return null; } var guid = GuidManager.NewDynamicGuid(); if (WorldObjectFactory.CreateWorldObject(weenie, guid) is not Container link) { error = $"weenie {Mod.Settings.VaultWcid} did not come back as a container"; ModManager.Log($"[{Mod.Name}] {error}", ModManager.LogLevel.Error); return null; } // Belt: whatever the weenie row says, a vault link must be able to hold its slots // and must hold no sub-containers (see Settings.AllowPacksAndFoci). link.ItemCapacity = (byte)Mod.Settings.SlotsPerContainer; link.ContainerCapacity = 0; // A fresh container has InventoryLoaded true from the weenie ctor; nothing to read. var ok = false; var done = new ManualResetEventSlim(false); link.SaveBiotaToDatabase(false); DatabaseManager.Shard.SaveBiota(link.Biota, link.BiotaDatabaseLock, result => { ok = result; done.Set(); }); if (!done.Wait(TimeSpan.FromSeconds(Mod.Settings.LoadTimeoutSeconds)) || !ok) { // The guid is leaked - nothing points at it and no row claims it - which is // the harmless direction. Nothing was told it belongs to this account. error = "the vault could not be written to the database"; ModManager.Log($"[{Mod.Name}] could not save a new vault link 0x{guid.Full:X8} for account {handle.AccountId}; no row was written and no item was moved.", ModManager.LogLevel.Error); return null; } if (!MuleDb.AddLink(handle.AccountId, handle.Links.Count, guid.Full, out var dbError)) { error = "the vault could not be recorded"; ModManager.Log($"[{Mod.Name}] vault link 0x{guid.Full:X8} saved but NOT recorded for account {handle.AccountId}: {dbError}. It is an empty orphan container - `mule-admin orphans` lists it. No item was moved.", ModManager.LogLevel.Error); return null; } handle.Links.Add(link); ModManager.Log($"[{Mod.Name}] account {handle.AccountId} vault grew to {handle.Links.Count} link(s); new link 0x{guid.Full:X8}"); return link; } // ---- KEEPING IT ON DISK -------------------------------------------------- /// /// Saves every loaded vault: the containers' own rows and any item still marked /// changed. Takes each account's lock, so it cannot read a container's Inventory while /// the world thread is moving something in or out of it. /// /// Called on a timer and - synchronously - from Dispose. ACE runs ModManager.Shutdown /// AFTER WorldManager.StopWorld, so anything queued onto the world thread at shutdown /// is dropped on the floor; the shard's own queue is still drained afterwards, so a /// direct save from here does land. /// public static int SaveAll(string why) { var biotas = new System.Collections.ObjectModel.Collection<(ACE.Entity.Models.Biota biota, ReaderWriterLockSlim rwLock)>(); var items = 0; foreach (var handle in _loaded.Values.ToList()) { if (handle.Fault.Length > 0) continue; using (Hold(handle.AccountId)) { foreach (var link in handle.Links) { Collect(link, biotas); foreach (var item in link.Inventory.Values.ToList()) { if (Collect(item, biotas)) items++; } } } } if (biotas.Count > 0) DatabaseManager.Shard.SaveBiotasInParallel(biotas, null); if (biotas.Count > 0) { Interlocked.Add(ref SweepSaves, biotas.Count); Mod.Trace($"{why}: {biotas.Count} vault row(s) written ({items} item(s))"); } return biotas.Count; } /// ACE's Landblock.AddWorldObjectToBiotasSaveCollection, in effect. private static bool Collect(WorldObject wo, System.Collections.ObjectModel.Collection<(ACE.Entity.Models.Biota biota, ReaderWriterLockSlim rwLock)> biotas) { if (!wo.ChangesDetected) return false; wo.SaveBiotaToDatabase(false); biotas.Add((wo.Biota, wo.BiotaDatabaseLock)); return true; } /// /// The timer: save everything, then let go of any vault whose account has nobody /// online. /// public static void Sweep() { try { SaveAll("sweep"); if (Mod.Settings.ReleaseAfterSeconds <= 0) return; var online = new HashSet(); foreach (var player in PlayerManager.GetAllOnline()) { var id = player?.Account?.AccountId ?? 0; if (id != 0) online.Add(id); } var cutoff = DateTime.UtcNow.AddSeconds(-Mod.Settings.ReleaseAfterSeconds); foreach (var handle in _loaded.Values.ToList()) { if (online.Contains(handle.AccountId) || handle.LastTouched > cutoff) continue; Release(handle.AccountId); } } catch (Exception ex) { ModManager.Log($"[{Mod.Name}] the vault sweep threw: {ex.Message}", ModManager.LogLevel.Error); } } /// /// Lets go of a loaded vault after saving it. /// /// DROPS REFERENCES ONLY. It must never call WorldObject.Destroy on a link or on /// anything in one: Destroy walks a container's Inventory destroying every item, /// removes their biotas from the shard and hands their guids back to GuidManager. That /// is how a vault would eat an account's gear, and it is the one call this mod must /// never make. /// public static bool Release(uint accountId) { if (!_loaded.TryGetValue(accountId, out var handle)) return false; using (Hold(accountId)) { var biotas = new System.Collections.ObjectModel.Collection<(ACE.Entity.Models.Biota biota, ReaderWriterLockSlim rwLock)>(); foreach (var link in handle.Links) { Collect(link, biotas); foreach (var item in link.Inventory.Values.ToList()) Collect(item, biotas); } if (biotas.Count > 0) DatabaseManager.Shard.SaveBiotasInParallel(biotas, null); _loaded.TryRemove(accountId, out _); } Mod.Trace($"vault of account {accountId} released"); return true; } /// Every loaded vault let go, after saving. Hot reload and shutdown. public static void ReleaseAll() { foreach (var accountId in _loaded.Keys.ToList()) Release(accountId); } }