namespace Aeshnidae.Mule;
///
/// One account's vault, in memory: the chain of container WorldObjects and the items in
/// them. Every item in here is the SAME WorldObject the player was carrying - the same
/// guid, the same biota, the same spells, tinks, enchantments and item XP. A deposit is a
/// change of container and nothing else. Nothing in this mod reads an item's properties
/// into a row and rebuilds the item later, and nothing ever may.
///
public sealed class VaultHandle
{
public uint AccountId { get; init; }
/// Chain links in seq order. Index == seq.
public List Links { get; } = new();
/// True once every link's inventory is back from the shard database.
public volatile bool Ready;
/// Set when a load failed; every verb refuses and says this.
public volatile string Fault = "";
public DateTime LastTouched = DateTime.UtcNow;
///
/// Items held, counted off the live containers. This is the authority while a vault is
/// loaded; MuleDb.CountInContainers is the disk's opinion and the two are compared by
/// `mule-admin verify`.
///
public int Count => Links.Sum(l => l.Inventory.Count);
public IEnumerable Items => Links.SelectMany(l => l.Inventory.Values);
}
///
/// Which vaults are loaded, how they are loaded, how they are kept on disk, and the lock
/// that makes two clients on one account safe.
///
/// WHY THIS IS NOT JUST A CONTAINER IN A LANDBLOCK
/// -----------------------------------------------
/// A vault container is in no landblock, and that is deliberate - it must never be
/// reachable, visible or openable. ACE's whole upkeep of an object runs off its landblock:
/// Landblock.Tick walks a sorted heartbeat list that an object only joins through
/// AddWorldObject, and Landblock.SaveDB only ever walks its own worldObjects. So a vault
/// container gets NO heartbeat and NO automatic save, and this mod has to do both jobs:
///
/// - saving: every movement forces the moved item's biota to disk before the player is
/// told it worked (Movement), a timer sweeps for retries, and Dispose saves
/// synchronously at shutdown.
/// - the heartbeat we deliberately do NOT replace: enchantment durations on a stored
/// item do not tick down, and an item with a Lifespan cannot expire in storage.
/// Both are recorded in the Readme as what the vault does.
///
/// Loading is ACE's own: Container's biota constructor asks the shard for everything whose
/// container_Id is the container, and queues the sort onto WorldManager's action queue -
/// which the world thread drains every tick whether or not any landblock is awake
/// (WorldManager.UpdateGameWorld -> ActionQueue.RunActions). WorldObject.EnqueueAction has
/// an explicit allowance for exactly this case. So the load works; it is just not instant,
/// and a vault that has not finished loading is never treated as empty.
///
public static class Vault
{
private static readonly ConcurrentDictionary _loaded = new();
private static readonly ConcurrentDictionary _gates = new();
private static readonly ConcurrentDictionary _loading = new();
/// How often a mutation had to wait for another one on the same account.
public static int LockWaits;
/// How many loads have been started, finished and failed, for mule-admin.
public static int LoadsStarted, LoadsFinished, LoadsFailed;
/// Items saved by the safety-net sweep, and rows it wrote.
public static int SweepSaves;
///
/// THE ACCOUNT LOCK. Every mutation of a vault, and every read that must agree with a
/// mutation (the count behind the limit, the sweep's walk of Inventory), is done inside
/// `lock (Vault.Gate(accountId))`.
///
/// Two clients on one account is normal here - FairPlay's per-address caps are off for
/// the beta - so two simultaneous deposits are the obvious race. Player commands
/// happen to arrive on the world thread (InboundMessageManager's queue is drained in
/// WorldManager.UpdateWorld), which would serialise them by itself; the console does
/// NOT - Aeshnidae.RemoteConsole invokes a command handler straight off its own watcher
/// thread - and nor does the save sweep. So the lock is load-bearing, not decoration,
/// and `mule-admin race` exists to prove it holds.
///
public static object Gate(uint accountId) => _gates.GetOrAdd(accountId, _ => new object());
/// Takes the account lock, counting the wait if there was one.
public static IDisposable Hold(uint accountId)
{
var gate = Gate(accountId);
if (!Monitor.TryEnter(gate))
{
Interlocked.Increment(ref LockWaits);
Monitor.Enter(gate);
}
return new Held(gate);
}
private sealed class Held : IDisposable
{
private readonly object _gate;
private bool _done;
public Held(object gate) => _gate = gate;
public void Dispose()
{
if (_done) return;
_done = true;
Monitor.Exit(_gate);
}
}
public static IEnumerable Loaded => _loaded.Values;
public static int LoadedCount => _loaded.Count;
///
/// The loaded vault for an account, or null with a reason. NEVER BLOCKS: this is called
/// from the world thread, and the load it would be waiting for finishes ON the world
/// thread, so waiting here would deadlock the server. A first call starts the load in
/// the background and refuses; by the time a player can type a second command it is
/// there. Login warms it (Patches), so in practice a player never sees the refusal.
///
public static VaultHandle? Get(uint accountId, out string error)
{
error = "";
if (!Mod.Settings.Enabled)
{
error = "The vault is closed.";
return null;
}
if (!MuleDb.Ready)
{
error = "The vault's ledger is not available right now. Nothing has been changed.";
return null;
}
if (_loaded.TryGetValue(accountId, out var handle))
{
handle.LastTouched = DateTime.UtcNow;
if (handle.Fault.Length > 0)
{
error = handle.Fault;
return null;
}
if (!handle.Ready)
{
error = "Your vault is still being read. Try again in a moment.";
return null;
}
return handle;
}
Warm(accountId);
error = "Your vault is being read. Try again in a moment.";
return null;
}
///
/// Is this still THE loaded vault for its account?
///
/// Get() cannot take the account lock - it is called from the world thread and the load
/// it would wait for finishes on that thread - so there is a sliver between a caller
/// getting a handle and taking the lock in which the sweep could have released that
/// vault. Nothing on disk could go wrong (the item's own row is still the truth, and the
/// sweep saves before letting go), but the released handle's in-memory count would
/// diverge from a freshly loaded one, and the limit is counted from that count. So every
/// mutation re-checks this as its first act inside the lock, and refuses if it lost the
/// race rather than working from a count that may be stale.
///
public static bool IsCurrent(VaultHandle handle) =>
handle is not null
&& _loaded.TryGetValue(handle.AccountId, out var current)
&& ReferenceEquals(current, handle);
///
/// Starts loading an account's vault if it is not loaded or loading. Safe to call from
/// any thread and as often as you like. Called on login, so the first /mule is ready.
///
public static void Warm(uint accountId)
{
if (!Mod.Settings.Enabled || !MuleDb.Ready || accountId == 0)
return;
if (_loaded.ContainsKey(accountId))
{
if (_loaded.TryGetValue(accountId, out var h))
h.LastTouched = DateTime.UtcNow;
return;
}
if (!_loading.TryAdd(accountId, 0))
return;
Interlocked.Increment(ref LoadsStarted);
Task.Run(() =>
{
try { LoadNow(accountId); }
catch (Exception ex)
{
Interlocked.Increment(ref LoadsFailed);
ModManager.Log($"[{Mod.Name}] loading the vault of account {accountId} threw: {ex}", ModManager.LogLevel.Error);
_loaded[accountId] = new VaultHandle
{
AccountId = accountId,
Fault = "Your vault could not be read. It has not been changed - tell a developer.",
};
}
finally { _loading.TryRemove(accountId, out _); }
});
}
///
/// Reads an account's chain off the shard and builds the live containers. Runs on a
/// background task - never the world thread - because GetBiota blocks and because the
/// sort of each container's contents has to run on the world thread while this waits.
///
private static void LoadNow(uint accountId)
{
var rows = MuleDb.Chain(accountId);
var handle = new VaultHandle { AccountId = accountId };
foreach (var row in rows)
{
var dbBiota = DatabaseManager.Shard.BaseDatabase.GetBiota(row.ContainerId);
if (dbBiota is null)
{
// The row names a container that is not in the biota table. Do NOT invent
// one: anything that was in it is pointing at a guid nothing owns, and
// re-minting the container with that guid is a repair an operator should
// choose (`mule-admin repair`), not something a login does silently.
handle.Fault = "Your vault could not be opened. Nothing has been changed - tell a developer.";
Interlocked.Increment(ref LoadsFailed);
ModManager.Log($"[{Mod.Name}] VAULT LINK MISSING: account {accountId} link {row.Seq} names container 0x{row.ContainerId:X8}, which has no biota row. The vault is refused until `mule-admin repair {accountId}` is run deliberately.",
ModManager.LogLevel.Error);
_loaded[accountId] = handle;
return;
}
if (dbBiota.WeenieClassId != Mod.Settings.VaultWcid)
{
// Worse: the guid belongs to something else now. Never touch it.
handle.Fault = "Your vault could not be opened. Nothing has been changed - tell a developer.";
Interlocked.Increment(ref LoadsFailed);
ModManager.Log($"[{Mod.Name}] VAULT GUID COLLISION: account {accountId} link {row.Seq} names 0x{row.ContainerId:X8}, but that biota is weenie {dbBiota.WeenieClassId}, not the vault weenie {Mod.Settings.VaultWcid}. Refusing the whole vault.",
ModManager.LogLevel.Error);
_loaded[accountId] = handle;
return;
}
if (WorldObjectFactory.CreateWorldObject(dbBiota) is not Container link)
{
handle.Fault = "Your vault could not be opened. Nothing has been changed - tell a developer.";
Interlocked.Increment(ref LoadsFailed);
ModManager.Log($"[{Mod.Name}] vault link 0x{row.ContainerId:X8} of account {accountId} did not come back as a Container.",
ModManager.LogLevel.Error);
_loaded[accountId] = handle;
return;
}
handle.Links.Add(link);
}
// Publish it unready so a command refuses rather than reading a half-loaded vault.
_loaded[accountId] = handle;
var deadline = DateTime.UtcNow.AddSeconds(Mod.Settings.LoadTimeoutSeconds);
while (handle.Links.Any(l => !l.InventoryLoaded))
{
if (DateTime.UtcNow > deadline)
{
handle.Fault = "Your vault is taking too long to read. Nothing has been changed - try again shortly.";
Interlocked.Increment(ref LoadsFailed);
ModManager.Log($"[{Mod.Name}] the vault of account {accountId} did not finish loading inside {Mod.Settings.LoadTimeoutSeconds}s ({handle.Links.Count(l => !l.InventoryLoaded)} of {handle.Links.Count} link(s) still reading).",
ModManager.LogLevel.Error);
return;
}
Thread.Sleep(25);
}
handle.Ready = true;
Interlocked.Increment(ref LoadsFinished);
Mod.Trace($"vault of account {accountId} loaded: {handle.Links.Count} link(s), {handle.Count} item(s)");
}
///
/// The link a deposit should go into: the first with a free slot. Null when the chain
/// is as long as MaxItems allows and every link is full - the caller turns that into
/// the limit refusal. Grows the chain when it needs to. Call under the account lock.
///
public static Container? LinkWithRoom(VaultHandle handle, out string error)
{
error = "";
foreach (var link in handle.Links)
{
if (link.Inventory.Count < Mod.Settings.SlotsPerContainer)
return link;
}
if (handle.Links.Count >= Mod.Settings.ChainLength)
{
error = "full";
return null;
}
return Grow(handle, out error);
}
///
/// Mints one more link: a container WorldObject of our hidden weenie, saved to the
/// shard and CONFIRMED there before the row that claims it is written.
///
/// The order is the safety. See MuleDb.AddLink for why this way round and not the
/// other. Called under the account lock, on the world thread, so it blocks the world
/// for one database round trip - once per 250 items an account ever deposits.
///
private static Container? Grow(VaultHandle handle, out string error)
{
error = "";
var weenie = DatabaseManager.World.GetCachedWeenie(Mod.Settings.VaultWcid);
if (weenie is null)
{
error = $"the vault weenie {Mod.Settings.VaultWcid} is not in the world database";
ModManager.Log($"[{Mod.Name}] {error} - no vault can be created", ModManager.LogLevel.Error);
return null;
}
var guid = GuidManager.NewDynamicGuid();
if (WorldObjectFactory.CreateWorldObject(weenie, guid) is not Container link)
{
error = $"weenie {Mod.Settings.VaultWcid} did not come back as a container";
ModManager.Log($"[{Mod.Name}] {error}", ModManager.LogLevel.Error);
return null;
}
// Belt: whatever the weenie row says, a vault link must be able to hold its slots
// and must hold no sub-containers (see Settings.AllowPacksAndFoci).
link.ItemCapacity = (byte)Mod.Settings.SlotsPerContainer;
link.ContainerCapacity = 0;
// A fresh container has InventoryLoaded true from the weenie ctor; nothing to read.
var ok = false;
var done = new ManualResetEventSlim(false);
link.SaveBiotaToDatabase(false);
DatabaseManager.Shard.SaveBiota(link.Biota, link.BiotaDatabaseLock, result =>
{
ok = result;
done.Set();
});
if (!done.Wait(TimeSpan.FromSeconds(Mod.Settings.LoadTimeoutSeconds)) || !ok)
{
// The guid is leaked - nothing points at it and no row claims it - which is
// the harmless direction. Nothing was told it belongs to this account.
error = "the vault could not be written to the database";
ModManager.Log($"[{Mod.Name}] could not save a new vault link 0x{guid.Full:X8} for account {handle.AccountId}; no row was written and no item was moved.",
ModManager.LogLevel.Error);
return null;
}
if (!MuleDb.AddLink(handle.AccountId, handle.Links.Count, guid.Full, out var dbError))
{
error = "the vault could not be recorded";
ModManager.Log($"[{Mod.Name}] vault link 0x{guid.Full:X8} saved but NOT recorded for account {handle.AccountId}: {dbError}. It is an empty orphan container - `mule-admin orphans` lists it. No item was moved.",
ModManager.LogLevel.Error);
return null;
}
handle.Links.Add(link);
ModManager.Log($"[{Mod.Name}] account {handle.AccountId} vault grew to {handle.Links.Count} link(s); new link 0x{guid.Full:X8}");
return link;
}
// ---- KEEPING IT ON DISK --------------------------------------------------
///
/// Saves every loaded vault: the containers' own rows and any item still marked
/// changed. Takes each account's lock, so it cannot read a container's Inventory while
/// the world thread is moving something in or out of it.
///
/// Called on a timer and - synchronously - from Dispose. ACE runs ModManager.Shutdown
/// AFTER WorldManager.StopWorld, so anything queued onto the world thread at shutdown
/// is dropped on the floor; the shard's own queue is still drained afterwards, so a
/// direct save from here does land.
///
public static int SaveAll(string why)
{
var biotas = new System.Collections.ObjectModel.Collection<(ACE.Entity.Models.Biota biota, ReaderWriterLockSlim rwLock)>();
var items = 0;
foreach (var handle in _loaded.Values.ToList())
{
if (handle.Fault.Length > 0)
continue;
using (Hold(handle.AccountId))
{
foreach (var link in handle.Links)
{
Collect(link, biotas);
foreach (var item in link.Inventory.Values.ToList())
{
if (Collect(item, biotas))
items++;
}
}
}
}
if (biotas.Count > 0)
DatabaseManager.Shard.SaveBiotasInParallel(biotas, null);
if (biotas.Count > 0)
{
Interlocked.Add(ref SweepSaves, biotas.Count);
Mod.Trace($"{why}: {biotas.Count} vault row(s) written ({items} item(s))");
}
return biotas.Count;
}
/// ACE's Landblock.AddWorldObjectToBiotasSaveCollection, in effect.
private static bool Collect(WorldObject wo,
System.Collections.ObjectModel.Collection<(ACE.Entity.Models.Biota biota, ReaderWriterLockSlim rwLock)> biotas)
{
if (!wo.ChangesDetected)
return false;
wo.SaveBiotaToDatabase(false);
biotas.Add((wo.Biota, wo.BiotaDatabaseLock));
return true;
}
///
/// The timer: save everything, then let go of any vault whose account has nobody
/// online.
///
public static void Sweep()
{
try
{
SaveAll("sweep");
if (Mod.Settings.ReleaseAfterSeconds <= 0)
return;
var online = new HashSet();
foreach (var player in PlayerManager.GetAllOnline())
{
var id = player?.Account?.AccountId ?? 0;
if (id != 0)
online.Add(id);
}
var cutoff = DateTime.UtcNow.AddSeconds(-Mod.Settings.ReleaseAfterSeconds);
foreach (var handle in _loaded.Values.ToList())
{
if (online.Contains(handle.AccountId) || handle.LastTouched > cutoff)
continue;
Release(handle.AccountId);
}
}
catch (Exception ex)
{
ModManager.Log($"[{Mod.Name}] the vault sweep threw: {ex.Message}", ModManager.LogLevel.Error);
}
}
///
/// Lets go of a loaded vault after saving it.
///
/// DROPS REFERENCES ONLY. It must never call WorldObject.Destroy on a link or on
/// anything in one: Destroy walks a container's Inventory destroying every item,
/// removes their biotas from the shard and hands their guids back to GuidManager. That
/// is how a vault would eat an account's gear, and it is the one call this mod must
/// never make.
///
public static bool Release(uint accountId)
{
if (!_loaded.TryGetValue(accountId, out var handle))
return false;
using (Hold(accountId))
{
var biotas = new System.Collections.ObjectModel.Collection<(ACE.Entity.Models.Biota biota, ReaderWriterLockSlim rwLock)>();
foreach (var link in handle.Links)
{
Collect(link, biotas);
foreach (var item in link.Inventory.Values.ToList())
Collect(item, biotas);
}
if (biotas.Count > 0)
DatabaseManager.Shard.SaveBiotasInParallel(biotas, null);
_loaded.TryRemove(accountId, out _);
}
Mod.Trace($"vault of account {accountId} released");
return true;
}
/// Every loaded vault let go, after saving. Hot reload and shutdown.
public static void ReleaseAll()
{
foreach (var accountId in _loaded.Keys.ToList())
Release(accountId);
}
}