using System.Security.Cryptography; namespace Aeshnidae.DatBackup; /// /// Snapshots of a folder of dat files, stored content-addressed. /// /// The shape is chosen by what dats actually are: four files, 1.4 GB together, that /// change rarely and never partially. Copying all of them per snapshot would cost a /// gigabyte to record that nothing happened, so each distinct file is stored once /// under its own SHA-256 and a snapshot is just a manifest naming the hashes it /// wants. A snapshot of an unchanged set therefore costs a few hundred bytes, which /// is what makes it reasonable to take one on every server start. /// /// Restoring is a copy out of the blob store, never a rename, so the store keeps /// every generation it has been asked to keep even after a restore. /// /// Deliberately plain BCL - no ACE types, no logging framework, no settings object - /// because the launcher links this same file to snapshot client dat profiles. The /// caller supplies a logger and the policy. /// public static class SnapshotStore { /// What counts as a dat set. Nothing else in the folder is recorded. public const string DatPattern = "*.dat"; public sealed class FileRecord { public string Name { get; set; } = ""; public long Size { get; set; } public string Sha256 { get; set; } = ""; public DateTime ModifiedUtc { get; set; } } public sealed class Snapshot { public string Id { get; set; } = ""; public DateTime TakenUtc { get; set; } public string Source { get; set; } = ""; public string Label { get; set; } = ""; public List Files { get; set; } = new(); public long TotalSize => Files.Sum(f => f.Size); } private static readonly JsonSerializerOptions Json = new() { WriteIndented = true, ReadCommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true, }; private static string BlobDir(string store) => Path.Combine(store, "blobs"); private static string SnapshotDir(string store) => Path.Combine(store, "snapshots"); /// /// Blobs are bucketed by the first two hex characters. One flat directory of a /// few thousand entries is fine on NTFS; the buckets cost nothing and keep it /// browsable by hand, which matters when someone is recovering a dat at 2am. /// private static string BlobPath(string store, string hash) => Path.Combine(BlobDir(store), hash[..2], hash); // ------------------------------------------------------------------ taking /// /// Records the current contents of . Returns the /// snapshot, or null with a reason if there was nothing to record. /// public static (Snapshot? Snapshot, string Message) Take( string store, string source, string label, int keep, Action? log = null) { log ??= _ => { }; if (string.IsNullOrWhiteSpace(source) || !Directory.Exists(source)) return (null, $"nothing to snapshot - {source} does not exist"); var dats = Directory.GetFiles(source, DatPattern).OrderBy(f => f).ToList(); if (dats.Count == 0) return (null, $"nothing to snapshot - no {DatPattern} in {source}"); Directory.CreateDirectory(SnapshotDir(store)); var snapshot = new Snapshot { Id = NextId(store), TakenUtc = DateTime.UtcNow, Source = source, Label = label, }; long stored = 0, deduped = 0; foreach (var path in dats) { var info = new FileInfo(path); string hash; try { hash = Hash(path); } catch (Exception ex) { // A dat held open by a running client or the server itself is the // normal case, not an error - but a snapshot missing a file is a lie, // so the whole thing is abandoned rather than silently partial. return (null, $"could not read {info.Name}: {ex.Message}"); } snapshot.Files.Add(new FileRecord { Name = info.Name, Size = info.Length, Sha256 = hash, ModifiedUtc = info.LastWriteTimeUtc, }); var blob = BlobPath(store, hash); if (File.Exists(blob)) { deduped += info.Length; continue; } Directory.CreateDirectory(Path.GetDirectoryName(blob)!); // Write beside and move, so an interrupted copy cannot leave a blob that // is named after a hash it does not have. var tmp = blob + ".partial"; File.Copy(path, tmp, overwrite: true); File.Move(tmp, blob, overwrite: true); stored += info.Length; } File.WriteAllText(Path.Combine(SnapshotDir(store), snapshot.Id + ".json"), JsonSerializer.Serialize(snapshot, Json)); var pruned = Prune(store, keep, log); var message = $"snapshot {snapshot.Id}: {snapshot.Files.Count} file(s), " + $"{Mb(stored)} MB new, {Mb(deduped)} MB already stored" + (pruned > 0 ? $", {pruned} old snapshot(s) pruned" : ""); log(message); return (snapshot, message); } /// /// A timestamp to the second, made unique by suffix if it has to be. /// /// The id is what an admin types at 2am, so it stays human - a sortable /// timestamp, not a guid. But a manual snapshot taken in the same second as the /// startup one would otherwise land on the same manifest filename and silently /// replace it, which is a backup quietly disappearing. Found by the tests. /// private static string NextId(string store) { var stamp = DateTime.Now.ToString("yyyyMMdd-HHmmss"); var id = stamp; for (var n = 2; File.Exists(Path.Combine(SnapshotDir(store), id + ".json")); n++) id = $"{stamp}-{n}"; return id; } private static string Hash(string path) { // Share everything: this reads files a client or the server may have open, // and reading is all it does. using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 1 << 20); using var sha = SHA256.Create(); return Convert.ToHexString(sha.ComputeHash(stream)).ToLowerInvariant(); } private static long Mb(long bytes) => bytes / 1024 / 1024; // ------------------------------------------------------------------ reading public static List List(string store) { var dir = SnapshotDir(store); if (!Directory.Exists(dir)) return new List(); var found = new List(); foreach (var file in Directory.GetFiles(dir, "*.json")) { try { var s = JsonSerializer.Deserialize(File.ReadAllText(file), Json); if (s is not null) found.Add(s); } catch { /* a manifest we cannot read is skipped, not fatal */ } } return found.OrderByDescending(s => s.TakenUtc).ToList(); } public static Snapshot? Find(string store, string id) => List(store).FirstOrDefault(s => s.Id.Equals(id, StringComparison.OrdinalIgnoreCase)); /// Newest first, so "the last known good one" is Latest(). public static Snapshot? Latest(string store) => List(store).FirstOrDefault(); // ---------------------------------------------------------------- verifying /// /// Confirms every blob a snapshot names is present and still hashes to its name. /// A backup nobody has verified is a hope, not a backup. /// public static (bool Ok, List Problems) Verify(string store, Snapshot snapshot) { var problems = new List(); foreach (var f in snapshot.Files) { var blob = BlobPath(store, f.Sha256); if (!File.Exists(blob)) { problems.Add($"{f.Name}: blob {f.Sha256[..12]} is missing"); continue; } if (new FileInfo(blob).Length != f.Size) { problems.Add($"{f.Name}: blob is the wrong size"); continue; } try { if (!Hash(blob).Equals(f.Sha256, StringComparison.OrdinalIgnoreCase)) problems.Add($"{f.Name}: blob does not match its hash"); } catch (Exception ex) { problems.Add($"{f.Name}: {ex.Message}"); } } return (problems.Count == 0, problems); } // ---------------------------------------------------------------- restoring /// /// Copies a snapshot's files back over . /// /// Checked before anything is written: every blob must be present, and every /// target must be writable. A dat held open by the server or a running client /// cannot be replaced, and finding that out half way through would leave a set /// that is neither the old one nor the new one. /// public static (bool Ok, string Message) Restore( string store, Snapshot snapshot, string destination, Action? log = null) { log ??= _ => { }; if (!Directory.Exists(destination)) return (false, $"{destination} does not exist"); foreach (var f in snapshot.Files) if (!File.Exists(BlobPath(store, f.Sha256))) return (false, $"cannot restore - the stored copy of {f.Name} is missing"); var locked = new List(); foreach (var f in snapshot.Files) { var target = Path.Combine(destination, f.Name); if (!File.Exists(target)) continue; try { using var probe = new FileStream(target, FileMode.Open, FileAccess.Write, FileShare.None); } catch { locked.Add(f.Name); } } if (locked.Count > 0) return (false, $"cannot restore - in use by a running client or the server: {string.Join(", ", locked)}. " + "Stop them and try again."); var restored = 0; foreach (var f in snapshot.Files) { var target = Path.Combine(destination, f.Name); File.Copy(BlobPath(store, f.Sha256), target, overwrite: true); File.SetLastWriteTimeUtc(target, f.ModifiedUtc); restored++; log($"restored {f.Name}"); } return (true, $"restored {restored} file(s) from snapshot {snapshot.Id} into {destination}"); } // ------------------------------------------------------------------ pruning /// /// Keeps the newest snapshots and deletes any blob no /// surviving snapshot refers to. Returns how many snapshots went. /// public static int Prune(string store, int keep, Action? log = null) { log ??= _ => { }; if (keep <= 0) return 0; var all = List(store); if (all.Count <= keep) return 0; var doomed = all.Skip(keep).ToList(); foreach (var s in doomed) { try { File.Delete(Path.Combine(SnapshotDir(store), s.Id + ".json")); } catch (Exception ex) { log($"could not remove snapshot {s.Id}: {ex.Message}"); } } // Only now is it safe to work out what is unreferenced. var live = List(store).SelectMany(s => s.Files).Select(f => f.Sha256) .ToHashSet(StringComparer.OrdinalIgnoreCase); if (Directory.Exists(BlobDir(store))) { foreach (var blob in Directory.GetFiles(BlobDir(store), "*", SearchOption.AllDirectories)) { var name = Path.GetFileName(blob); if (name.EndsWith(".partial", StringComparison.OrdinalIgnoreCase) || !live.Contains(name)) { try { File.Delete(blob); } catch (Exception ex) { log($"could not remove blob {name[..Math.Min(12, name.Length)]}: {ex.Message}"); } } } } return doomed.Count; } /// Bytes the blob store is actually using, for the status command. public static long StoreSize(string store) { if (!Directory.Exists(BlobDir(store))) return 0; return Directory.GetFiles(BlobDir(store), "*", SearchOption.AllDirectories) .Sum(f => new FileInfo(f).Length); } }